diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1e853e87f3d974f04c1f5950e50acb89db4354e2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..65d997569cbd0fb0a97f2c0b99429d6f4f7a1a0f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9939e0e1d2df3b6dd179ab08ac6c38f93c18a081 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..8c484f6b80abe63b9c6f9f31607a850ce076e34d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8922668c8e20f35ffbc131cbd09fa2295e279585 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..68c69c70a714b8e5677de6f2abd33781395a70ad Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6c6cd1f3139f1e199d2ffb133ba5540b6b9192a5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..f8031ee4ba1b3b8ebe608a5887bab4ce7f69127d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..87763f11f348ba63ac12c7926f38f718fca5c20b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..5fd60124430b6eeb462c0ea6c082bbebe55f9a5a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5df651ba62ee883a393eebabacb2e2c778121582 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b8fb220c80b65769b059459726da4b65872449e1 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/plots_step_35000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/train/events.out.tfevents.1794732175.workergpu169.1879625.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/train/events.out.tfevents.1794732175.workergpu169.1879625.0 new file mode 100644 index 0000000000000000000000000000000000000000..ce4bf1544b4d2a7a324ae1af974720b5eb353241 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/train/events.out.tfevents.1794732175.workergpu169.1879625.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c50baf116b141825ae5a297ff9db7cb9f09fd79da73376d387e22df074684ddc +size 1288219 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/valid/events.out.tfevents.1794732175.workergpu169.1879625.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/valid/events.out.tfevents.1794732175.workergpu169.1879625.1 new file mode 100644 index 0000000000000000000000000000000000000000..56c507dd8cdf996377d176d037a32a886c621907 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/runs/valid/events.out.tfevents.1794732175.workergpu169.1879625.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9fa7337d571a547223966facbc27cb6c032ff8818d1e72655abeb3109753aa9 +size 17950124 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..72ff1be6108880536a1e07f8817f1cf8b2a9b91d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_15mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca954f3dbfa464e74c05874231c2314564e41b87 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6d2cdc61e225eedeb1436f814ae70eca0745d024 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log @@ -0,0 +1,2765 @@ +rank0 [2026-11-15 03:42:52,687] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-15 03:42:52,687] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 03:42:53,119] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:42:53,119] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:42:53,120] INFO : Instantiating model +rank0 [2026-11-15 03:42:53,142] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:42:53,143] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:42:53,144] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:42:53,144] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:42:53,144] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:42:53,144] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:42:53,144] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,147] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:42:53,147] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,150] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:42:53,150] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,152] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:42:53,152] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,155] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:42:53,155] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,157] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:42:53,158] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,160] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 03:42:53,160] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:42:53,160] INFO : Initializing output DNNs +rank0 [2026-11-15 03:42:53,161] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,164] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,171] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:42:53,171] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:42:53,171] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:42:53,171] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:42:53,171] INFO : task_queries enabled=True +rank0 [2026-11-15 03:42:53,171] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:42:53,171] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:42:53,171] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:42:53,171] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:42:53,171] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:42:53,171] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:42:53,171] INFO : output_mode=elementwise +rank0 [2026-11-15 03:42:53,171] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:42:53,172] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:42:53,172] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:42:53,172] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:42:53,173] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:42:53,173] INFO : MLPF __init__ done +rank0 [2026-11-15 03:42:53,173] INFO : Instantiated model +rank0 [2026-11-15 03:42:53,174] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:42:53,182] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:42:53,183] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:42:53,305] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:42:53,306] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:42:53,306] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:42:53,399] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:42:53,399] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:42:53,401] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:42:53,401] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:42:53,401] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:42:53,403] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:42:53,408] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046 +rank0 [2026-11-15 03:42:53,408] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046 +rank0 [2026-11-15 03:42:53,408] INFO : Getting dataloaders +rank0 [2026-11-15 03:42:53,408] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:54,612] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,659] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,702] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,744] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,781] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,812] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,852] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,882] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,938] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,982] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,983] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:42:54,983] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:42:54,983] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:42:54,983] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:54,990] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:54,997] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,004] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,012] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,019] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,026] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,033] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,040] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,047] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,054] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,054] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:42:55,054] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:42:55,101] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:55,234] INFO : Got dataloaders +rank0 [2026-11-15 03:42:55,234] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:42:55,234] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:42:55,234] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:42:55,243] INFO : Creating train_iterator +rank0 [2026-11-15 03:42:55,244] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:55,268] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:55,393] INFO : Created train_iterator +rank0 [2026-11-15 03:43:07,394] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9870172393321992, 'Classification': 0.016035858988761902, 'Regression_pt': 0.09887309789657593, 'Regression_eta': 0.006165142094250769, 'Regression_sin_phi': 0.005006907158531249, 'Regression_cos_phi': 0.0051970563700888305, 'Regression_energy': 0.09819029666483402}, weights={'Classification_binary': 1.0, 'Classification': 123.91087341308594, 'Regression_pt': 20.096641540527344, 'Regression_eta': 322.2987060546875, 'Regression_sin_phi': 396.855224609375, 'Regression_cos_phi': 382.33514404296875, 'Regression_energy': 20.236391067504883} +rank0 [2026-11-15 03:43:57,269] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:43:57,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:44,189] INFO : Step 1000/500000 rank0 | Train Loss: 11.6175 | Train Loss Unweighted: 2.1554 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9803 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0730 | Regression_eta: 0.0054 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6093 | Classification_binary: 1.9803 | Regression_cos_phi: 1.4906 | Regression_energy: 1.4782 | Regression_eta: 1.7275 | Regression_pt: 1.5126 | Regression_sin_phi: 1.8190 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0024 | max_backward: 0.0404 | max_optimizer_step: 0.0567 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2609.0 | uss_MB: 1373.4 | child_rss_MB: 22794.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 03:44:44,190] INFO : Running validation on rank0 +rank0 [2026-11-15 03:44:44,190] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:44:44,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:44:44,373] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:44:44,555] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:44:44,635] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:44:44,689] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:44:44,743] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:44:44,790] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:44:44,844] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:44:44,900] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:44:44,949] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:44:45,025] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:44:45,104] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:44:45,156] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:44:45,210] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:44:45,254] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:44:45,307] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:44:45,360] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:44:45,409] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:44:45,413] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:44:45,415] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:44:45,418] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:44:45,419] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:44:45,421] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:44:45,422] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:44:45,425] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:44:45,427] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:44:45,429] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:44:45,430] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:44:45,433] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:44:45,435] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:44:45,436] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:44:45,438] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:44:45,441] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:44:46,948] INFO : VALIDATION | Step=1000/500000 | Train Loss=11.6175 | Valid Loss=13.8300 | Train Loss Unweighted=2.1554 | Valid Loss Unweighted=2.2064 | Stale=0 | ETA=929.0m +rank0 [2026-11-15 03:44:46,948] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:44:47,022] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:44:47,029] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:47,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:44:47,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:44:47,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:44:48,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:44:48,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:44:48,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:44:48,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:44:49,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:44:49,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:44:49,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:44:50,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:44:50,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:44:50,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:44:50,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:44:50,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:44:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:44:51,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:44:51,533] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:44:51,541] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:52,103] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:52,820] INFO : Plotted number of elements +rank0 [2026-11-15 03:44:54,177] INFO : Plotted jets +rank0 [2026-11-15 03:44:55,543] INFO : Plotted jet ratio +rank0 [2026-11-15 03:45:09,059] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:45:09,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:08,518] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:46:08,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:41,981] INFO : Step 2000/500000 rank0 | Train Loss: 11.2152 | Train Loss Unweighted: 1.8050 | Train Loss Components: Classification: 0.0138 | Classification_binary: 1.6116 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0839 | Regression_eta: 0.0031 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.7091 | Classification_binary: 1.6116 | Regression_cos_phi: 2.0506 | Regression_energy: 1.6977 | Regression_eta: 0.9979 | Regression_pt: 1.6789 | Regression_sin_phi: 1.4693 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4593s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0394 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.5 | uss_MB: 1553.7 | child_rss_MB: 22797.3 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 49.6 +rank0 [2026-11-15 03:46:41,983] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:46:42,052] INFO : Running validation on rank0 +rank0 [2026-11-15 03:46:42,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:46:42,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:42,212] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:46:42,331] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:42,445] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:42,512] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:42,576] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:42,639] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:42,703] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:42,768] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:42,872] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:42,984] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:43,097] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:43,164] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:43,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:43,292] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:43,356] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:43,420] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:43,532] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:43,539] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:43,541] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:43,542] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:43,545] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:43,547] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:43,548] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:43,551] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:43,553] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:43,555] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:43,557] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:43,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:43,561] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:43,563] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:43,565] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:43,567] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:45,017] INFO : VALIDATION | Step=2000/500000 | Train Loss=11.2152 | Valid Loss=13.8985 | Train Loss Unweighted=1.8050 | Valid Loss Unweighted=2.2112 | Stale=1 | ETA=953.6m +rank0 [2026-11-15 03:46:45,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:46:45,090] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:46:45,096] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:45,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:45,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:45,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:46,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:46,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:46,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:46,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:47,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:47,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:47,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:47,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:48,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:48,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:48,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:48,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:49,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:49,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:49,356] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:46:49,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:49,923] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:50,313] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:52,007] INFO : Plotted jets +rank0 [2026-11-15 03:46:52,990] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:19,537] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:47:19,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:19,142] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:48:19,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:39,659] INFO : Step 3000/500000 rank0 | Train Loss: 15.6563 | Train Loss Unweighted: 2.1858 | Train Loss Components: Classification: 0.0150 | Classification_binary: 1.9451 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1012 | Regression_eta: 0.0088 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.8567 | Classification_binary: 1.9451 | Regression_cos_phi: 1.8755 | Regression_energy: 2.0471 | Regression_eta: 2.8304 | Regression_pt: 2.0734 | Regression_sin_phi: 3.0282 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2834.2 | uss_MB: 1607.6 | child_rss_MB: 22796.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 03:48:39,660] INFO : Running validation on rank0 +rank0 [2026-11-15 03:48:39,660] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:48:39,660] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:48:39,806] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:48:39,931] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:48:40,046] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:48:40,115] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:48:40,181] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:48:40,246] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:48:40,313] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:48:40,379] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:48:40,484] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:48:40,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:48:40,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:48:40,772] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:48:40,837] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:48:40,901] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:48:40,966] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:48:41,031] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:48:41,143] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:48:41,150] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:48:41,152] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:48:41,153] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:48:41,155] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:48:41,157] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:48:41,159] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:48:41,161] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:48:41,163] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:48:41,165] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:48:41,168] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:48:41,170] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:48:41,172] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:48:41,174] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:48:41,176] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:48:41,177] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:48:42,620] INFO : VALIDATION | Step=3000/500000 | Train Loss=15.6563 | Valid Loss=13.8469 | Train Loss Unweighted=2.1858 | Valid Loss Unweighted=2.2132 | Stale=2 | ETA=959.1m +rank0 [2026-11-15 03:48:42,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:48:42,696] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:48:42,712] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:42,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:48:43,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:48:43,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:48:43,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:48:43,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:48:44,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:48:44,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:48:44,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:48:44,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:48:45,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:48:45,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:48:45,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:48:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:48:46,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:48:46,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:48:46,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:48:46,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:48:46,986] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:48:46,991] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:47,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:47,927] INFO : Plotted number of elements +rank0 [2026-11-15 03:48:49,634] INFO : Plotted jets +rank0 [2026-11-15 03:48:50,609] INFO : Plotted jet ratio +rank0 [2026-11-15 03:49:30,553] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:49:30,554] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:30,429] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:50:30,429] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:37,810] INFO : Step 4000/500000 rank0 | Train Loss: 14.9054 | Train Loss Unweighted: 2.1119 | Train Loss Components: Classification: 0.0197 | Classification_binary: 1.9076 | Regression_cos_phi: 0.0097 | Regression_energy: 0.0844 | Regression_eta: 0.0037 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.4465 | Classification_binary: 1.9076 | Regression_cos_phi: 3.6956 | Regression_energy: 1.7070 | Regression_eta: 1.1852 | Regression_pt: 1.6267 | Regression_sin_phi: 2.3368 | LR: 2.08e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4510s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0344 | max_loss: 0.0024 | max_backward: 0.0386 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2866.4 | uss_MB: 1639.8 | child_rss_MB: 22801.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:50:37,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:50:37,876] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:37,876] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:37,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:38,020] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:50:38,136] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:38,251] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:38,319] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:38,385] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:38,450] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:38,517] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,584] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:38,803] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:38,912] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:38,978] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:39,043] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:39,107] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:39,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,236] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,347] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:39,354] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:39,356] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:39,359] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:39,361] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:39,361] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:39,363] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:39,366] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:39,367] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:39,369] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:39,371] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:39,374] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:39,375] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:39,378] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:39,380] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:39,381] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:40,884] INFO : VALIDATION | Step=4000/500000 | Train Loss=14.9054 | Valid Loss=13.8275 | Train Loss Unweighted=2.1119 | Valid Loss Unweighted=2.2141 | Stale=0 | ETA=962.3m +rank0 [2026-11-15 03:50:40,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:50:40,957] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:50:40,963] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:41,013] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:41,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:41,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:42,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:42,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:42,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:42,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:43,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:43,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:43,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:43,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:44,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:44,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:44,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:44,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:45,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:45,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:45,289] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:50:45,294] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:45,853] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:46,243] INFO : Plotted number of elements +rank0 [2026-11-15 03:50:47,942] INFO : Plotted jets +rank0 [2026-11-15 03:50:48,913] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:41,957] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:51:41,957] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:52:35,779] INFO : Step 5000/500000 rank0 | Train Loss: 12.6854 | Train Loss Unweighted: 2.1025 | Train Loss Components: Classification: 0.0124 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1073 | Regression_eta: 0.0038 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5405 | Classification_binary: 1.8557 | Regression_cos_phi: 2.0577 | Regression_energy: 2.1720 | Regression_eta: 1.2281 | Regression_pt: 2.2905 | Regression_sin_phi: 1.5410 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0593 | max_tensorboard_logging: 0.1260 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2885.3 | uss_MB: 1658.8 | child_rss_MB: 22802.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:52:35,780] INFO : Running validation on rank0 +rank0 [2026-11-15 03:52:35,780] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:52:35,780] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:52:35,926] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:52:36,041] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:52:36,157] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:52:36,225] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:52:36,290] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:52:36,356] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:52:36,423] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:52:36,490] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:52:36,595] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:52:36,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:52:36,821] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:52:36,887] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:52:36,952] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:52:37,015] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:52:37,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:52:37,147] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:52:37,256] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:52:37,262] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:52:37,265] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:52:37,267] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:52:37,269] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:52:37,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:52:37,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:52:37,275] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:52:37,277] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:52:37,279] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:52:37,281] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:52:37,283] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:52:37,285] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:52:37,287] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:52:37,289] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:52:37,291] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:52:38,795] INFO : VALIDATION | Step=5000/500000 | Train Loss=12.6854 | Valid Loss=13.8205 | Train Loss Unweighted=2.1025 | Valid Loss Unweighted=2.2168 | Stale=0 | ETA=962.9m +rank0 [2026-11-15 03:52:38,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:52:38,870] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:52:38,876] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:38,925] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:52:39,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:52:39,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:52:39,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:52:40,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:52:40,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:52:40,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:52:40,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:52:41,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:52:41,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:52:41,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:52:41,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:52:42,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:52:42,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:52:42,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:52:42,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:52:43,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:52:43,194] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:52:43,201] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:43,764] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:44,146] INFO : Plotted number of elements +rank0 [2026-11-15 03:52:45,451] INFO : Plotted jets +rank0 [2026-11-15 03:52:46,833] INFO : Plotted jet ratio +rank0 [2026-11-15 03:52:53,337] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:52:53,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:53,322] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:53:53,322] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:54:34,362] INFO : Step 6000/500000 rank0 | Train Loss: 9.6100 | Train Loss Unweighted: 2.0251 | Train Loss Components: Classification: 0.0145 | Classification_binary: 1.8540 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0737 | Regression_eta: 0.0025 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.7945 | Classification_binary: 1.8540 | Regression_cos_phi: 0.8898 | Regression_energy: 1.4906 | Regression_eta: 0.7922 | Regression_pt: 1.5066 | Regression_sin_phi: 1.2822 | LR: 2.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4704s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1279 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.6 | uss_MB: 1603.1 | child_rss_MB: 22806.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 03:54:34,364] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:54:34,431] INFO : Running validation on rank0 +rank0 [2026-11-15 03:54:34,431] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:54:34,431] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:54:34,575] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:54:34,694] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:54:34,806] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:54:34,875] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:54:34,940] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:54:35,005] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:54:35,073] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:54:35,140] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:54:35,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:54:35,360] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:54:35,474] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:54:35,541] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:54:35,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:54:35,672] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:54:35,737] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:54:35,802] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:54:35,911] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:54:35,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:54:35,920] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:54:35,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:54:35,924] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:54:35,926] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:54:35,927] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:54:35,930] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:54:35,932] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:54:35,934] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:54:35,936] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:54:35,938] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:54:35,939] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:54:35,942] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:54:35,944] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:54:35,946] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:54:37,391] INFO : VALIDATION | Step=6000/500000 | Train Loss=9.6100 | Valid Loss=13.8499 | Train Loss Unweighted=2.0251 | Valid Loss Unweighted=2.2220 | Stale=1 | ETA=963.5m +rank0 [2026-11-15 03:54:37,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:37,465] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:54:37,472] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:37,521] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:38,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:54:38,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:54:38,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:54:38,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:54:39,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:54:39,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:54:39,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:54:39,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:54:40,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:54:40,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:54:40,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:54:40,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:54:41,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:54:41,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:54:41,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:54:41,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:54:41,791] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:54:41,796] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:42,364] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:42,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:54:44,435] INFO : Plotted jets +rank0 [2026-11-15 03:54:45,408] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:05,004] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:55:05,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:56:05,242] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:56:05,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:56:32,915] INFO : Step 7000/500000 rank0 | Train Loss: 14.5605 | Train Loss Unweighted: 1.9631 | Train Loss Components: Classification: 0.0178 | Classification_binary: 1.7586 | Regression_cos_phi: 0.0078 | Regression_energy: 0.0815 | Regression_eta: 0.0051 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.2046 | Classification_binary: 1.7586 | Regression_cos_phi: 2.9650 | Regression_energy: 1.6496 | Regression_eta: 1.6587 | Regression_pt: 1.7235 | Regression_sin_phi: 2.6006 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4681s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1262 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2940.6 | uss_MB: 1714.2 | child_rss_MB: 22804.1 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:56:32,916] INFO : Running validation on rank0 +rank0 [2026-11-15 03:56:32,916] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:56:32,916] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:56:33,064] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:56:33,181] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:56:33,293] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:56:33,361] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:56:33,426] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:56:33,491] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:56:33,557] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:56:33,622] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:56:33,728] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:56:33,844] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:56:33,958] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:56:34,026] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:56:34,094] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:56:34,160] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:56:34,225] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:56:34,289] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:56:34,397] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:56:34,402] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:56:34,404] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:56:34,407] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:56:34,409] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:56:34,410] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:56:34,414] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:56:34,416] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:56:34,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:56:34,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:56:34,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:56:34,422] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:56:34,426] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:56:34,426] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:56:34,428] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:56:34,430] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:56:35,870] INFO : VALIDATION | Step=7000/500000 | Train Loss=14.5605 | Valid Loss=13.8935 | Train Loss Unweighted=1.9631 | Valid Loss Unweighted=2.2243 | Stale=2 | ETA=963.3m +rank0 [2026-11-15 03:56:35,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:56:35,944] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:56:35,950] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:35,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:56:36,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:56:36,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:56:36,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:56:37,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:56:37,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:56:37,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:56:37,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:56:38,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:56:38,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:56:38,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:56:39,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:56:39,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:56:39,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:56:39,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:56:39,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:56:40,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:56:40,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:56:40,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:40,871] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:41,252] INFO : Plotted number of elements +rank0 [2026-11-15 03:56:42,959] INFO : Plotted jets +rank0 [2026-11-15 03:56:43,929] INFO : Plotted jet ratio +rank0 [2026-11-15 03:57:16,810] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:57:16,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:16,899] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:58:16,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:31,349] INFO : Step 8000/500000 rank0 | Train Loss: 13.5738 | Train Loss Unweighted: 2.1991 | Train Loss Components: Classification: 0.0137 | Classification_binary: 1.9968 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0851 | Regression_eta: 0.0063 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6962 | Classification_binary: 1.9968 | Regression_cos_phi: 1.8861 | Regression_energy: 1.7217 | Regression_eta: 2.0379 | Regression_pt: 1.7254 | Regression_sin_phi: 2.5097 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4754s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0427 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1204 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2963.3 | uss_MB: 1736.8 | child_rss_MB: 22804.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:58:31,351] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:58:31,416] INFO : Running validation on rank0 +rank0 [2026-11-15 03:58:31,416] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:58:31,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:58:31,559] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:58:31,677] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:58:31,788] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:58:31,855] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:58:31,920] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:58:31,984] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:58:32,051] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:58:32,117] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:58:32,222] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:58:32,336] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:58:32,451] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:58:32,517] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:58:32,583] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:58:32,649] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:58:32,715] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:58:32,782] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:58:32,891] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:58:32,898] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:58:32,898] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:58:32,901] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:58:32,904] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:58:32,906] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:58:32,907] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:58:32,910] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:58:32,912] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:58:32,914] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:58:32,916] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:58:32,918] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:58:32,920] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:58:32,922] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:58:32,924] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:58:32,926] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:58:34,379] INFO : VALIDATION | Step=8000/500000 | Train Loss=13.5738 | Valid Loss=13.8437 | Train Loss Unweighted=2.1991 | Valid Loss Unweighted=2.2264 | Stale=3 | ETA=962.6m +rank0 [2026-11-15 03:58:34,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:58:34,451] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:58:34,457] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:34,506] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:58:34,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:58:35,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:58:35,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:58:35,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:58:36,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:58:36,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:58:36,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:58:36,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:58:37,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:58:37,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:58:37,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:58:37,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:58:38,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:58:38,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:58:38,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:58:38,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:58:38,774] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:58:38,782] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:39,357] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:39,737] INFO : Plotted number of elements +rank0 [2026-11-15 03:58:41,448] INFO : Plotted jets +rank0 [2026-11-15 03:58:42,431] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:28,658] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:59:28,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:28,693] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:00:28,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:29,936] INFO : Step 9000/500000 rank0 | Train Loss: 12.3439 | Train Loss Unweighted: 2.1709 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.9652 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0892 | Regression_eta: 0.0034 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5895 | Classification_binary: 1.9652 | Regression_cos_phi: 1.9989 | Regression_energy: 1.8048 | Regression_eta: 1.0958 | Regression_pt: 1.8040 | Regression_sin_phi: 2.0856 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1265 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2903.4 | uss_MB: 1677.0 | child_rss_MB: 22804.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:00:29,937] INFO : Running validation on rank0 +rank0 [2026-11-15 04:00:29,937] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:00:29,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:00:30,086] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:00:30,206] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:00:30,319] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:00:30,388] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:00:30,452] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:00:30,517] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:00:30,583] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:00:30,649] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:00:30,756] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:00:30,872] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:00:30,987] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:00:31,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:00:31,123] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:00:31,188] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:00:31,254] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:00:31,320] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:00:31,430] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:00:31,437] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:00:31,439] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:00:31,441] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:00:31,443] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:00:31,445] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:00:31,446] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:00:31,448] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:00:31,450] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:00:31,452] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:00:31,455] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:00:31,456] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:00:31,458] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:00:31,462] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:00:31,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:00:31,466] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:32,917] INFO : VALIDATION | Step=9000/500000 | Train Loss=12.3439 | Valid Loss=13.8528 | Train Loss Unweighted=2.1709 | Valid Loss Unweighted=2.2233 | Stale=4 | ETA=961.7m +rank0 [2026-11-15 04:00:32,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:00:32,991] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:00:32,999] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:33,049] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:33,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:33,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:34,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:34,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:34,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:34,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:35,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:35,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:35,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:35,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:36,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:36,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:36,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:36,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:37,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:37,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:37,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:00:37,294] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:37,863] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:38,244] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:39,941] INFO : Plotted jets +rank0 [2026-11-15 04:00:40,917] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:40,321] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:01:40,321] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:28,457] INFO : Step 10000/500000 rank0 | Train Loss: 12.3633 | Train Loss Unweighted: 2.1192 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.9369 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0784 | Regression_eta: 0.0052 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4968 | Classification_binary: 1.9369 | Regression_cos_phi: 2.1570 | Regression_energy: 1.5867 | Regression_eta: 1.6890 | Regression_pt: 1.5279 | Regression_sin_phi: 1.9690 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4852s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.9 | uss_MB: 1785.4 | child_rss_MB: 22804.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:02:28,459] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:02:28,525] INFO : Running validation on rank0 +rank0 [2026-11-15 04:02:28,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:02:28,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:02:28,669] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:02:28,787] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:02:28,902] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:02:28,969] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:02:29,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:02:29,103] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:02:29,170] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:02:29,237] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:02:29,344] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:02:29,459] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:02:29,574] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:02:29,642] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:02:29,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:02:29,773] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:02:29,839] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:02:29,904] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:02:30,013] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:02:30,020] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:02:30,022] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:02:30,023] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:02:30,027] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:02:30,027] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:02:30,030] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:02:30,033] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:02:30,034] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:02:30,037] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:02:30,039] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:02:30,041] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:02:30,043] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:02:30,045] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:02:30,047] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:02:30,050] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:02:31,515] INFO : VALIDATION | Step=10000/500000 | Train Loss=12.3633 | Valid Loss=13.9266 | Train Loss Unweighted=2.1192 | Valid Loss Unweighted=2.2295 | Stale=5 | ETA=960.6m +rank0 [2026-11-15 04:02:31,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:02:31,590] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:02:31,594] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:31,644] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:02:32,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:02:32,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:02:32,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:02:32,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:02:33,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:02:33,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:02:33,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:02:33,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:02:34,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:02:34,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:02:34,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:02:34,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:02:35,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:02:35,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:02:35,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:02:35,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:02:35,960] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:02:35,968] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:36,533] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:36,912] INFO : Plotted number of elements +rank0 [2026-11-15 04:02:38,639] INFO : Plotted jets +rank0 [2026-11-15 04:02:39,630] INFO : Plotted jet ratio +rank0 [2026-11-15 04:02:52,344] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:02:52,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:03:52,438] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:03:52,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:27,174] INFO : Step 11000/500000 rank0 | Train Loss: 12.0314 | Train Loss Unweighted: 2.1470 | Train Loss Components: Classification: 0.0153 | Classification_binary: 1.9321 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0918 | Regression_eta: 0.0034 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.8947 | Classification_binary: 1.9321 | Regression_cos_phi: 1.5363 | Regression_energy: 1.8579 | Regression_eta: 1.0929 | Regression_pt: 1.9270 | Regression_sin_phi: 1.7905 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1249 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3037.0 | uss_MB: 1810.6 | child_rss_MB: 22811.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:04:27,175] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:27,175] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:27,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:27,320] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:04:27,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:27,549] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:27,617] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:27,682] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:27,747] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:27,812] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:27,878] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:27,984] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:28,102] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:28,217] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:28,285] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:28,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:28,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:28,482] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:28,547] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:28,658] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:28,664] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:28,667] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:28,668] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:28,671] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:28,673] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:28,676] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:28,677] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:28,679] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:28,681] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:28,683] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:28,684] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:28,687] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:28,688] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:28,690] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:28,694] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:30,155] INFO : VALIDATION | Step=11000/500000 | Train Loss=12.0314 | Valid Loss=14.0317 | Train Loss Unweighted=2.1470 | Valid Loss Unweighted=2.2364 | Stale=6 | ETA=959.4m +rank0 [2026-11-15 04:04:30,156] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:04:30,231] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:04:30,237] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:30,287] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:30,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:31,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:31,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:31,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:31,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:32,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:32,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:32,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:32,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:33,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:33,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:33,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:33,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:34,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:34,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:34,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:34,582] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:04:34,600] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:35,168] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:35,548] INFO : Plotted number of elements +rank0 [2026-11-15 04:04:36,854] INFO : Plotted jets +rank0 [2026-11-15 04:04:38,237] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:04,117] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:05:04,117] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:06:03,944] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:06:03,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:06:25,648] INFO : Step 12000/500000 rank0 | Train Loss: 12.6323 | Train Loss Unweighted: 1.8929 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.7227 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0702 | Regression_eta: 0.0088 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5966 | Classification_binary: 1.7227 | Regression_cos_phi: 1.7871 | Regression_energy: 1.4207 | Regression_eta: 2.8234 | Regression_pt: 1.3846 | Regression_sin_phi: 1.8972 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0410 | max_loss: 0.0027 | max_backward: 0.0407 | max_optimizer_step: 0.0613 | max_tensorboard_logging: 0.1274 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3057.9 | uss_MB: 1831.4 | child_rss_MB: 22808.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:06:25,650] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:06:25,715] INFO : Running validation on rank0 +rank0 [2026-11-15 04:06:25,715] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:06:25,715] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:06:25,862] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:06:25,980] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:06:26,094] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:06:26,162] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:06:26,228] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:06:26,292] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:06:26,357] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:06:26,422] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:06:26,527] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:06:26,641] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:06:26,757] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:06:26,823] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:06:26,889] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:06:26,956] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:06:27,024] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:06:27,092] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:06:27,200] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:06:27,207] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:06:27,208] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:06:27,210] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:06:27,212] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:06:27,216] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:06:27,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:06:27,218] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:06:27,220] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:06:27,222] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:06:27,224] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:06:27,228] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:06:27,230] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:06:27,232] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:06:27,232] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:06:27,236] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:06:28,685] INFO : VALIDATION | Step=12000/500000 | Train Loss=12.6323 | Valid Loss=14.1165 | Train Loss Unweighted=1.8929 | Valid Loss Unweighted=2.2412 | Stale=7 | ETA=958.0m +rank0 [2026-11-15 04:06:28,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:06:28,762] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:06:28,767] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:28,818] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:06:29,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:06:29,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:06:29,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:06:30,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:06:30,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:06:30,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:06:30,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:06:31,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:06:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:06:31,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:06:31,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:06:32,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:06:32,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:06:32,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:06:32,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:06:33,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:06:33,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:06:33,103] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:33,674] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:34,056] INFO : Plotted number of elements +rank0 [2026-11-15 04:06:35,355] INFO : Plotted jets +rank0 [2026-11-15 04:06:36,735] INFO : Plotted jet ratio +rank0 [2026-11-15 04:07:15,955] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:07:15,955] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:16,373] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:08:16,373] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:24,685] INFO : Step 13000/500000 rank0 | Train Loss: 10.4660 | Train Loss Unweighted: 1.8990 | Train Loss Components: Classification: 0.0114 | Classification_binary: 1.7438 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0647 | Regression_eta: 0.0065 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4160 | Classification_binary: 1.7438 | Regression_cos_phi: 1.4441 | Regression_energy: 1.3083 | Regression_eta: 2.0837 | Regression_pt: 1.3254 | Regression_sin_phi: 1.1447 | LR: 2.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4671s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0607 | max_tensorboard_logging: 0.1220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3090.5 | uss_MB: 1864.0 | child_rss_MB: 22805.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:08:24,686] INFO : Running validation on rank0 +rank0 [2026-11-15 04:08:24,687] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:08:24,687] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:08:24,832] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:08:24,948] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:08:25,060] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:08:25,130] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:08:25,195] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:08:25,261] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:08:25,327] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:08:25,392] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:08:25,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:08:25,613] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:08:25,727] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:08:25,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:08:25,860] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:08:25,924] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:08:25,988] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:08:26,054] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:08:26,163] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:08:26,170] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:08:26,172] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:08:26,174] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:08:26,176] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:08:26,178] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:08:26,180] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:08:26,182] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:08:26,184] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:08:26,186] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:08:26,188] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:08:26,190] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:08:26,192] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:08:26,194] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:08:26,196] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:08:26,198] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:08:27,635] INFO : VALIDATION | Step=13000/500000 | Train Loss=10.4660 | Valid Loss=14.1912 | Train Loss Unweighted=1.8990 | Valid Loss Unweighted=2.2452 | Stale=8 | ETA=956.8m +rank0 [2026-11-15 04:08:27,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:08:27,709] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:08:27,715] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:27,765] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:08:28,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:08:28,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:08:28,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:08:29,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:08:29,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:08:29,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:08:29,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:08:30,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:08:30,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:08:30,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:08:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:08:31,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:08:31,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:08:31,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:08:31,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:08:32,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:08:32,062] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:08:32,067] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:32,637] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:33,017] INFO : Plotted number of elements +rank0 [2026-11-15 04:08:34,331] INFO : Plotted jets +rank0 [2026-11-15 04:08:35,736] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:28,240] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:09:28,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:23,589] INFO : Step 14000/500000 rank0 | Train Loss: 8.5803 | Train Loss Unweighted: 1.8046 | Train Loss Components: Classification: 0.0119 | Classification_binary: 1.6694 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0572 | Regression_eta: 0.0023 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4747 | Classification_binary: 1.6694 | Regression_cos_phi: 1.4999 | Regression_energy: 1.1569 | Regression_eta: 0.7374 | Regression_pt: 1.1588 | Regression_sin_phi: 0.8832 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4784s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3114.0 | uss_MB: 1887.5 | child_rss_MB: 22811.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 15.6 +rank0 [2026-11-15 04:10:23,590] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:10:23,658] INFO : Running validation on rank0 +rank0 [2026-11-15 04:10:23,658] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:10:23,658] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:10:23,804] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:10:23,923] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:10:24,038] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:10:24,107] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:10:24,172] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:10:24,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:10:24,306] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:10:24,375] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:10:24,482] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:10:24,597] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:10:24,713] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:10:24,781] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:10:24,847] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:10:24,912] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:10:24,980] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:10:25,046] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:10:25,156] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:10:25,163] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:10:25,164] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:10:25,166] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:10:25,168] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:10:25,170] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:10:25,173] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:10:25,176] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:10:25,176] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:10:25,178] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:10:25,182] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:10:25,182] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:10:25,184] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:10:25,186] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:10:25,190] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:10:25,191] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:10:26,647] INFO : VALIDATION | Step=14000/500000 | Train Loss=8.5803 | Valid Loss=14.2052 | Train Loss Unweighted=1.8046 | Valid Loss Unweighted=2.2524 | Stale=9 | ETA=955.5m +rank0 [2026-11-15 04:10:26,647] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:10:26,720] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:10:26,724] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:26,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:10:27,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:10:27,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:10:27,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:10:28,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:10:28,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:10:28,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:10:28,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:10:29,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:10:29,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:10:29,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:10:29,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:10:30,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:10:30,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:10:30,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:10:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:10:31,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:10:31,068] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:10:31,073] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:31,646] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:32,026] INFO : Plotted number of elements +rank0 [2026-11-15 04:10:33,762] INFO : Plotted jets +rank0 [2026-11-15 04:10:34,784] INFO : Plotted jet ratio +rank0 [2026-11-15 04:10:40,518] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:10:40,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:40,344] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:11:40,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:22,014] INFO : Step 15000/500000 rank0 | Train Loss: 10.8163 | Train Loss Unweighted: 2.1410 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.9850 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0652 | Regression_eta: 0.0053 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6046 | Classification_binary: 1.9850 | Regression_cos_phi: 1.8253 | Regression_energy: 1.3200 | Regression_eta: 1.7023 | Regression_pt: 1.3090 | Regression_sin_phi: 1.0701 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4692s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3139.8 | uss_MB: 1913.3 | child_rss_MB: 22808.1 | allocated_MB: 170.9 | reserved_MB: 674.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:12:22,016] INFO : Running validation on rank0 +rank0 [2026-11-15 04:12:22,016] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:12:22,016] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:12:22,161] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:12:22,277] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:12:22,392] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:12:22,461] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:12:22,527] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:12:22,594] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:12:22,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:12:22,729] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:12:22,835] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:12:22,948] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:12:23,063] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:12:23,130] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:12:23,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:12:23,258] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:12:23,324] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:12:23,389] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:12:23,496] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:12:23,503] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:12:23,505] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:12:23,507] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:12:23,508] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:12:23,510] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:12:23,514] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:12:23,514] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:12:23,516] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:12:23,519] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:12:23,521] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:12:23,523] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:12:23,524] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:12:23,526] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:12:23,529] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:12:23,531] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:12:24,983] INFO : VALIDATION | Step=15000/500000 | Train Loss=10.8163 | Valid Loss=14.2968 | Train Loss Unweighted=2.1410 | Valid Loss Unweighted=2.2472 | Stale=10 | ETA=953.7m +rank0 [2026-11-15 04:12:24,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:12:25,070] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:12:25,073] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:25,120] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:12:25,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:12:25,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:12:26,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:12:26,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:12:26,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:12:26,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:12:27,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:12:27,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:12:27,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:12:27,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:12:28,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:12:28,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:12:28,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:12:28,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:12:29,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:12:29,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:12:29,418] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:12:29,426] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:29,992] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:30,376] INFO : Plotted number of elements +rank0 [2026-11-15 04:12:31,689] INFO : Plotted jets +rank0 [2026-11-15 04:12:33,103] INFO : Plotted jet ratio +rank0 [2026-11-15 04:12:51,965] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:12:51,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:51,852] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:13:51,852] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:14:20,423] INFO : Step 16000/500000 rank0 | Train Loss: 10.7147 | Train Loss Unweighted: 2.0555 | Train Loss Components: Classification: 0.0103 | Classification_binary: 1.8865 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0734 | Regression_eta: 0.0054 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.2765 | Classification_binary: 1.8865 | Regression_cos_phi: 1.6796 | Regression_energy: 1.4858 | Regression_eta: 1.7471 | Regression_pt: 1.4577 | Regression_sin_phi: 1.1816 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4794s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3163.1 | uss_MB: 1936.7 | child_rss_MB: 22811.7 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:14:20,425] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:14:20,491] INFO : Running validation on rank0 +rank0 [2026-11-15 04:14:20,491] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:14:20,491] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:14:20,636] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:14:20,752] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:14:20,865] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:14:20,932] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:14:20,998] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:14:21,065] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:14:21,133] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:14:21,199] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:14:21,306] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:14:21,421] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:14:21,536] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:14:21,605] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:14:21,670] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:14:21,733] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:14:21,799] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:14:21,864] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:14:21,972] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:14:21,979] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:14:21,981] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:14:21,982] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:14:21,986] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:14:21,986] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:14:21,988] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:14:21,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:14:21,992] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:14:21,996] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:14:21,997] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:14:21,999] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:14:22,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:14:22,004] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:14:22,006] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:14:22,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:14:23,470] INFO : VALIDATION | Step=16000/500000 | Train Loss=10.7147 | Valid Loss=14.4796 | Train Loss Unweighted=2.0555 | Valid Loss Unweighted=2.2542 | Stale=11 | ETA=952.0m +rank0 [2026-11-15 04:14:23,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:14:23,546] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:14:23,553] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:23,603] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:24,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:24,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:24,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:24,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:25,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:25,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:25,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:25,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:26,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:26,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:26,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:26,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:27,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:27,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:27,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:27,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:27,904] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:14:27,909] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,483] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,864] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:30,593] INFO : Plotted jets +rank0 [2026-11-15 04:14:31,579] INFO : Plotted jet ratio +rank0 [2026-11-15 04:15:03,668] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:15:03,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:04,265] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:16:04,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:19,574] INFO : Step 17000/500000 rank0 | Train Loss: 11.4339 | Train Loss Unweighted: 2.0598 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.8778 | Regression_cos_phi: 0.0046 | Regression_energy: 0.0776 | Regression_eta: 0.0032 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6430 | Classification_binary: 1.8778 | Regression_cos_phi: 1.7495 | Regression_energy: 1.5713 | Regression_eta: 1.0262 | Regression_pt: 1.5726 | Regression_sin_phi: 1.9935 | LR: 3.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.4734s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1314 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3101.2 | uss_MB: 1874.8 | child_rss_MB: 22811.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:16:19,575] INFO : Running validation on rank0 +rank0 [2026-11-15 04:16:19,575] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:16:19,575] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:16:19,720] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:16:19,840] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:16:19,952] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:16:20,022] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:16:20,089] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:16:20,153] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:16:20,218] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:16:20,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:16:20,390] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:16:20,506] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:16:20,620] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:16:20,687] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:16:20,756] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:16:20,820] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:16:20,884] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:16:20,949] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:16:21,059] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:16:21,066] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:16:21,067] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:16:21,070] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:16:21,072] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:16:21,074] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:16:21,076] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:16:21,078] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:16:21,080] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:16:21,082] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:16:21,083] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:16:21,086] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:16:21,087] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:16:21,089] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:16:21,092] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:16:21,095] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:16:22,530] INFO : VALIDATION | Step=17000/500000 | Train Loss=11.4339 | Valid Loss=14.5255 | Train Loss Unweighted=2.0598 | Valid Loss Unweighted=2.2621 | Stale=12 | ETA=950.5m +rank0 [2026-11-15 04:16:22,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:16:22,605] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:16:22,620] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:22,670] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:16:23,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:16:23,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:16:23,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:16:23,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:16:24,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:16:24,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:16:24,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:16:24,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:16:25,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:16:25,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:16:25,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:16:25,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:16:26,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:16:26,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:16:26,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:16:26,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:16:27,000] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:16:27,008] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:27,580] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:27,960] INFO : Plotted number of elements +rank0 [2026-11-15 04:16:29,280] INFO : Plotted jets +rank0 [2026-11-15 04:16:30,687] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:16,035] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:17:16,035] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:16,128] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:18:16,129] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:18,241] INFO : Step 18000/500000 rank0 | Train Loss: 11.3289 | Train Loss Unweighted: 2.1826 | Train Loss Components: Classification: 0.0130 | Classification_binary: 2.0009 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0788 | Regression_eta: 0.0055 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6136 | Classification_binary: 2.0009 | Regression_cos_phi: 1.5928 | Regression_energy: 1.5951 | Regression_eta: 1.7632 | Regression_pt: 1.5500 | Regression_sin_phi: 1.2131 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4815s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3206.4 | uss_MB: 1979.9 | child_rss_MB: 22810.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:18:18,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:18:18,310] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:18,310] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:18,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:18,455] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:18:18,574] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:18,688] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:18,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:18,824] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:18,889] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:18,956] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:19,024] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:19,132] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:19,248] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:19,363] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:19,429] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:19,494] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:19,558] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:19,623] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:19,688] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:19,798] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:19,806] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:19,808] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:19,810] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:19,812] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:19,814] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:19,815] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:19,818] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:19,820] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:19,822] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:19,824] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:19,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:19,828] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:19,830] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:19,832] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:19,834] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:21,299] INFO : VALIDATION | Step=18000/500000 | Train Loss=11.3289 | Valid Loss=14.4844 | Train Loss Unweighted=2.1826 | Valid Loss Unweighted=2.2684 | Stale=13 | ETA=948.9m +rank0 [2026-11-15 04:18:21,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:18:21,375] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:18:21,379] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:21,430] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:21,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:22,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:22,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:22,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:22,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:23,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:23,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:23,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:23,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:24,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:24,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:24,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:24,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:25,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:25,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:25,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:25,717] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:18:25,722] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:26,295] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:26,673] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:27,980] INFO : Plotted jets +rank0 [2026-11-15 04:18:29,404] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:27,897] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:19:27,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:16,598] INFO : Step 19000/500000 rank0 | Train Loss: 13.5408 | Train Loss Unweighted: 2.1380 | Train Loss Components: Classification: 0.0175 | Classification_binary: 1.8788 | Regression_cos_phi: 0.0028 | Regression_energy: 0.1129 | Regression_eta: 0.0068 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.1726 | Classification_binary: 1.8788 | Regression_cos_phi: 1.0526 | Regression_energy: 2.2848 | Regression_eta: 2.2060 | Regression_pt: 2.3115 | Regression_sin_phi: 1.6345 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1255 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3235.4 | uss_MB: 2009.0 | child_rss_MB: 22807.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 04:20:16,599] INFO : Running validation on rank0 +rank0 [2026-11-15 04:20:16,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:20:16,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:20:16,744] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:20:16,861] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:20:16,974] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:20:17,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:20:17,111] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:20:17,175] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:20:17,241] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:20:17,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:20:17,412] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:20:17,529] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:20:17,643] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:20:17,711] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:20:17,778] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:20:17,845] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:20:17,912] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:20:17,979] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:20:18,092] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:20:18,098] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:20:18,100] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:20:18,102] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:20:18,104] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:20:18,108] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:20:18,110] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:20:18,112] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:20:18,114] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:20:18,114] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:20:18,116] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:20:18,120] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:20:18,122] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:20:18,122] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:20:18,124] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:20:18,127] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:20:19,587] INFO : VALIDATION | Step=19000/500000 | Train Loss=13.5408 | Valid Loss=14.6540 | Train Loss Unweighted=2.1380 | Valid Loss Unweighted=2.2740 | Stale=14 | ETA=947.0m +rank0 [2026-11-15 04:20:19,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:20:19,662] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:20:19,665] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:19,716] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:20:20,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:20:20,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:20:20,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:20:21,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:20:21,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:20:21,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:20:21,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:20:22,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:20:22,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:20:22,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:20:22,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:20:23,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:20:23,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:20:23,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:20:23,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:20:24,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:20:24,067] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:20:24,076] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:24,651] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:25,033] INFO : Plotted number of elements +rank0 [2026-11-15 04:20:26,342] INFO : Plotted jets +rank0 [2026-11-15 04:20:27,758] INFO : Plotted jet ratio +rank0 [2026-11-15 04:20:39,670] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:20:39,670] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:39,967] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:21:39,968] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:15,756] INFO : Step 20000/500000 rank0 | Train Loss: 11.8463 | Train Loss Unweighted: 2.2414 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.0211 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0984 | Regression_eta: 0.0045 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5114 | Classification_binary: 2.0211 | Regression_cos_phi: 1.3833 | Regression_energy: 1.9910 | Regression_eta: 1.4344 | Regression_pt: 1.9637 | Regression_sin_phi: 1.5413 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4832s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0025 | max_backward: 0.0389 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3175.0 | uss_MB: 1948.6 | child_rss_MB: 22810.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 15.6 +rank0 [2026-11-15 04:22:15,758] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:22:15,824] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:15,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:15,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:22:15,978] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:22:16,102] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:22:16,219] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:22:16,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:22:16,357] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:22:16,424] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:22:16,491] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:22:16,557] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:22:16,662] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:22:16,775] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:22:16,886] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:22:16,953] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:22:17,019] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:22:17,085] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:22:17,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:22:17,215] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:22:17,324] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:22:17,330] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:22:17,333] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:22:17,335] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:22:17,337] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:22:17,339] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:22:17,340] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:22:17,343] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:22:17,345] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:22:17,347] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:22:17,349] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:22:17,351] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:22:17,353] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:22:17,355] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:22:17,357] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:22:17,360] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:22:18,813] INFO : VALIDATION | Step=20000/500000 | Train Loss=11.8463 | Valid Loss=14.8133 | Train Loss Unweighted=2.2414 | Valid Loss Unweighted=2.2824 | Stale=15 | ETA=945.4m +rank0 [2026-11-15 04:22:18,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:22:18,892] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:22:18,896] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:18,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:22:19,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:22:19,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:22:19,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:22:20,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:22:20,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:22:20,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:22:20,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:22:21,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:22:21,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:22:21,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:22:21,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:22:22,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:22:22,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:22:22,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:22:22,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:22:23,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:22:23,228] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:22:23,234] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:23,795] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:24,173] INFO : Plotted number of elements +rank0 [2026-11-15 04:22:25,479] INFO : Plotted jets +rank0 [2026-11-15 04:22:26,892] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:51,814] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:22:51,814] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:51,934] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:23:51,934] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:14,450] INFO : Step 21000/500000 rank0 | Train Loss: 11.0032 | Train Loss Unweighted: 2.2319 | Train Loss Components: Classification: 0.0078 | Classification_binary: 2.0639 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0700 | Regression_eta: 0.0055 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 0.9611 | Classification_binary: 2.0639 | Regression_cos_phi: 1.6690 | Regression_energy: 1.4162 | Regression_eta: 1.7701 | Regression_pt: 1.5365 | Regression_sin_phi: 1.5866 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3288.1 | uss_MB: 2061.7 | child_rss_MB: 22812.6 | allocated_MB: 170.4 | reserved_MB: 676.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:24:14,451] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:14,451] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:14,451] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:14,597] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:24:14,716] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:14,830] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:14,900] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:14,966] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:15,033] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:15,101] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:15,167] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:15,275] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:15,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:15,506] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:15,573] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:15,639] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:15,703] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:15,769] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:15,834] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:15,944] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:15,951] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:15,953] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:15,954] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:15,957] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:15,958] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:15,962] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:15,963] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:15,965] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:15,968] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:15,968] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:15,970] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:15,972] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:15,976] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:15,978] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:15,978] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:17,443] INFO : VALIDATION | Step=21000/500000 | Train Loss=11.0032 | Valid Loss=14.8529 | Train Loss Unweighted=2.2319 | Valid Loss Unweighted=2.2808 | Stale=16 | ETA=943.6m +rank0 [2026-11-15 04:24:17,444] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:17,520] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:17,524] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:17,575] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:18,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:18,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:18,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:18,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:19,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:19,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:19,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:19,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:20,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:20,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:20,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:20,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:21,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:21,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:21,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:21,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:21,865] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:24:21,873] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:22,440] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:22,819] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:24,149] INFO : Plotted jets +rank0 [2026-11-15 04:24:25,593] INFO : Plotted jet ratio +rank0 [2026-11-15 04:25:03,894] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:25:03,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:04,149] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:26:04,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:13,370] INFO : Step 22000/500000 rank0 | Train Loss: 10.2378 | Train Loss Unweighted: 2.2843 | Train Loss Components: Classification: 0.0111 | Classification_binary: 2.0957 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0842 | Regression_eta: 0.0025 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3747 | Classification_binary: 2.0957 | Regression_cos_phi: 1.0333 | Regression_energy: 1.7029 | Regression_eta: 0.8172 | Regression_pt: 1.6936 | Regression_sin_phi: 1.5204 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4913s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3312.7 | uss_MB: 2086.2 | child_rss_MB: 22809.2 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:26:13,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:26:13,439] INFO : Running validation on rank0 +rank0 [2026-11-15 04:26:13,439] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:26:13,439] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:26:13,584] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:26:13,699] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:26:13,811] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:26:13,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:26:13,944] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:26:14,008] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:26:14,076] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:26:14,141] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:26:14,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:26:14,358] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:26:14,470] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:26:14,537] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:26:14,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:26:14,668] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:26:14,735] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:26:14,801] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:26:14,914] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:26:14,921] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:26:14,921] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:26:14,924] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:26:14,926] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:26:14,929] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:26:14,931] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:26:14,933] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:26:14,935] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:26:14,937] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:26:14,939] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:26:14,940] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:26:14,943] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:26:14,945] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:26:14,947] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:26:14,949] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:26:16,396] INFO : VALIDATION | Step=22000/500000 | Train Loss=10.2378 | Valid Loss=14.9806 | Train Loss Unweighted=2.2843 | Valid Loss Unweighted=2.2841 | Stale=17 | ETA=941.9m +rank0 [2026-11-15 04:26:16,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:26:16,471] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:26:16,476] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:16,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:26:17,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:26:17,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:26:17,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:26:17,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:26:18,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:26:18,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:26:18,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:26:18,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:26:19,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:26:19,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:26:19,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:26:19,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:26:20,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:26:20,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:26:20,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:26:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:26:20,824] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:26:20,835] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:21,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:21,799] INFO : Plotted number of elements +rank0 [2026-11-15 04:26:23,554] INFO : Plotted jets +rank0 [2026-11-15 04:26:24,552] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:16,339] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:27:16,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:12,146] INFO : Step 23000/500000 rank0 | Train Loss: 8.0623 | Train Loss Unweighted: 1.9944 | Train Loss Components: Classification: 0.0111 | Classification_binary: 1.8564 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0589 | Regression_eta: 0.0013 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3803 | Classification_binary: 1.8564 | Regression_cos_phi: 0.6644 | Regression_energy: 1.1923 | Regression_eta: 0.4261 | Regression_pt: 1.2384 | Regression_sin_phi: 1.3046 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4814s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0024 | max_backward: 0.0385 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3323.2 | uss_MB: 2096.7 | child_rss_MB: 22811.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:28:12,148] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:12,148] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:12,148] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:12,294] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:28:12,413] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:12,526] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:12,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:12,662] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:12,728] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:12,794] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:12,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:12,968] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:13,087] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:13,203] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:13,269] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:13,335] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:13,399] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:13,465] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:13,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:13,642] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:13,649] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:13,650] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:13,653] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:13,655] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:13,657] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:13,659] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:13,661] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:13,663] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:13,666] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:13,666] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:13,669] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:13,670] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:13,672] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:13,674] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:13,677] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:15,129] INFO : VALIDATION | Step=23000/500000 | Train Loss=8.0623 | Valid Loss=15.1393 | Train Loss Unweighted=1.9944 | Valid Loss Unweighted=2.2871 | Stale=18 | ETA=940.1m +rank0 [2026-11-15 04:28:15,130] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:15,206] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:15,209] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:15,260] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:15,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:16,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:16,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:16,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:16,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:17,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:17,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:17,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:17,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:18,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:18,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:18,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:18,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:19,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:19,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:19,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:19,565] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:28:19,576] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:20,144] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:20,524] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:21,838] INFO : Plotted jets +rank0 [2026-11-15 04:28:23,266] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:28,110] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:28:28,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:27,981] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:29:27,982] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:10,491] INFO : Step 24000/500000 rank0 | Train Loss: 10.3893 | Train Loss Unweighted: 1.9571 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.8108 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0589 | Regression_eta: 0.0060 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5902 | Classification_binary: 1.8108 | Regression_cos_phi: 1.2409 | Regression_energy: 1.1920 | Regression_eta: 1.9329 | Regression_pt: 1.2440 | Regression_sin_phi: 1.3785 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5087s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1454 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3358.9 | uss_MB: 2132.5 | child_rss_MB: 22809.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:10,493] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:30:10,559] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:10,559] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:10,560] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:10,704] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:30:10,822] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:10,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:11,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:11,070] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:11,136] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:11,201] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:11,266] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:11,371] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:11,483] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:11,596] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:11,664] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:11,730] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:11,796] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:11,863] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:11,930] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:12,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:12,050] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:12,052] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:12,054] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:12,056] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:12,058] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:12,061] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:12,062] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:12,064] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:12,067] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:12,069] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:12,071] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:12,073] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:12,075] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:12,077] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:12,080] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:13,539] INFO : VALIDATION | Step=24000/500000 | Train Loss=10.3893 | Valid Loss=15.0848 | Train Loss Unweighted=1.9571 | Valid Loss Unweighted=2.3022 | Stale=19 | ETA=938.2m +rank0 [2026-11-15 04:30:13,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:13,614] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:13,617] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:13,669] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:14,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:14,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:14,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:14,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:15,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:15,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:15,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:15,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:16,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:16,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:16,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:16,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:17,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:17,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:17,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:17,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:17,971] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:30:17,977] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:18,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:18,936] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:20,225] INFO : Plotted jets +rank0 [2026-11-15 04:30:21,644] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:39,688] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:30:39,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:39,245] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:31:39,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:08,418] INFO : Step 25000/500000 rank0 | Train Loss: 10.9924 | Train Loss Unweighted: 2.0393 | Train Loss Components: Classification: 0.0157 | Classification_binary: 1.8905 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0616 | Regression_eta: 0.0058 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.9481 | Classification_binary: 1.8905 | Regression_cos_phi: 1.6976 | Regression_energy: 1.2467 | Regression_eta: 1.8607 | Regression_pt: 1.1720 | Regression_sin_phi: 1.1767 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3385.9 | uss_MB: 2159.5 | child_rss_MB: 22813.5 | allocated_MB: 170.4 | reserved_MB: 676.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:32:08,420] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:08,420] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:08,420] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:08,563] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:32:08,684] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:08,796] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:08,864] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:08,929] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:08,993] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:09,060] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:09,126] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:09,232] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:09,348] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:09,463] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:09,531] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:09,598] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:09,664] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:09,731] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:09,797] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:09,910] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:09,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:09,919] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:09,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:09,922] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:09,924] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:09,926] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:09,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:09,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:09,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:09,934] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:09,936] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:09,939] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:09,940] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:09,944] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:09,946] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:11,400] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.9924 | Valid Loss=15.2046 | Train Loss Unweighted=2.0393 | Valid Loss Unweighted=2.3036 | Stale=20 | ETA=936.1m +rank0 [2026-11-15 04:32:11,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:11,476] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:11,481] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:11,532] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:12,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:12,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:12,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:12,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:13,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:13,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:13,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:13,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:14,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:14,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:14,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:15,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:15,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:15,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:15,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:15,845] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:32:15,851] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:16,430] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:16,813] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:18,139] INFO : Plotted jets +rank0 [2026-11-15 04:32:19,559] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:51,046] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:32:51,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:50,751] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:33:50,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:06,986] INFO : Step 26000/500000 rank0 | Train Loss: 11.4067 | Train Loss Unweighted: 2.0718 | Train Loss Components: Classification: 0.0131 | Classification_binary: 1.9035 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0707 | Regression_eta: 0.0020 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6285 | Classification_binary: 1.9035 | Regression_cos_phi: 1.5703 | Regression_energy: 1.4306 | Regression_eta: 0.6343 | Regression_pt: 1.4346 | Regression_sin_phi: 2.8050 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3406.3 | uss_MB: 2179.9 | child_rss_MB: 22811.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:06,988] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:34:07,053] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:07,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:07,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:07,200] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:34:07,317] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:07,430] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:07,498] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:07,563] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:07,629] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:07,696] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:07,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:07,869] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:07,985] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:08,103] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:08,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:08,235] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:08,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:08,365] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:08,430] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:08,539] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:08,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:08,547] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:08,551] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:08,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:08,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:08,557] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:08,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:08,561] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:08,563] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:08,565] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:08,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:08,569] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:08,571] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:08,573] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:08,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:10,020] INFO : VALIDATION | Step=26000/500000 | Train Loss=11.4067 | Valid Loss=15.3003 | Train Loss Unweighted=2.0718 | Valid Loss Unweighted=2.3100 | Stale=21 | ETA=934.3m +rank0 [2026-11-15 04:34:10,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:10,092] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:10,095] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:10,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:10,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:10,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:11,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:11,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:11,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:11,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:12,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:12,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:12,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:12,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:13,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:13,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:13,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:13,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:14,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:14,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:14,465] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:34:14,471] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:15,048] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:15,429] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:16,734] INFO : Plotted jets +rank0 [2026-11-15 04:34:18,163] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:02,577] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:35:02,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:02,728] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:36:02,728] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:05,688] INFO : Step 27000/500000 rank0 | Train Loss: 10.5402 | Train Loss Unweighted: 1.9638 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.8033 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0690 | Regression_eta: 0.0071 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5016 | Classification_binary: 1.8033 | Regression_cos_phi: 1.5321 | Regression_energy: 1.3961 | Regression_eta: 2.2731 | Regression_pt: 1.3385 | Regression_sin_phi: 0.6956 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0402 | max_optimizer_step: 0.0630 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3432.1 | uss_MB: 2205.6 | child_rss_MB: 22812.4 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:36:05,690] INFO : Running validation on rank0 +rank0 [2026-11-15 04:36:05,690] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:36:05,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:36:05,834] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:36:05,951] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:36:06,065] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:36:06,134] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:36:06,200] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:36:06,264] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:36:06,329] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:36:06,394] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:36:06,499] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:36:06,612] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:36:06,727] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:36:06,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:36:06,862] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:36:06,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:36:06,994] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:36:07,063] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:36:07,176] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:36:07,184] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:36:07,184] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:36:07,187] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:36:07,190] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:36:07,191] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:36:07,194] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:36:07,196] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:36:07,198] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:36:07,200] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:36:07,202] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:36:07,204] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:36:07,206] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:36:07,208] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:36:07,210] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:36:07,212] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:36:08,648] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.5402 | Valid Loss=15.4868 | Train Loss Unweighted=1.9638 | Valid Loss Unweighted=2.3059 | Stale=22 | ETA=932.4m +rank0 [2026-11-15 04:36:08,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:36:08,723] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:36:08,726] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:08,778] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:36:09,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:36:09,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:36:09,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:36:10,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:36:10,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:36:10,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:36:10,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:36:11,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:36:11,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:36:11,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:36:11,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:36:12,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:36:12,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:36:12,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:36:12,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:36:12,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:36:13,040] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:36:13,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:13,618] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:14,002] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:15,290] INFO : Plotted jets +rank0 [2026-11-15 04:36:16,708] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:14,018] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:37:14,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:03,465] INFO : Step 28000/500000 rank0 | Train Loss: 9.7251 | Train Loss Unweighted: 2.0203 | Train Loss Components: Classification: 0.0131 | Classification_binary: 1.8789 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0589 | Regression_eta: 0.0028 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6242 | Classification_binary: 1.8789 | Regression_cos_phi: 1.4336 | Regression_energy: 1.1916 | Regression_eta: 0.9122 | Regression_pt: 1.1868 | Regression_sin_phi: 1.4978 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3455.0 | uss_MB: 2228.6 | child_rss_MB: 22812.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 04:38:03,467] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:38:03,533] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:03,533] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:03,533] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:03,685] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:38:03,806] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:03,923] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:03,994] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:04,062] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:04,129] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:04,197] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:04,264] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:04,369] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:04,484] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:04,596] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:04,664] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:04,729] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:04,793] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:04,858] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:04,922] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:05,033] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:05,039] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:05,042] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:05,043] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:05,047] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:05,047] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:05,050] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:05,053] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:05,055] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:05,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:05,059] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:05,060] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:05,063] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:05,065] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:05,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:05,069] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:06,511] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.7251 | Valid Loss=15.5625 | Train Loss Unweighted=2.0203 | Valid Loss Unweighted=2.3225 | Stale=23 | ETA=930.3m +rank0 [2026-11-15 04:38:06,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:06,593] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:06,596] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:06,644] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:07,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:07,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:07,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:07,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:08,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:08,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:08,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:08,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:09,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:09,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:09,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:09,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:10,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:10,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:10,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:10,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:10,954] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:38:10,960] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:11,527] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:11,910] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:13,232] INFO : Plotted jets +rank0 [2026-11-15 04:38:14,663] INFO : Plotted jet ratio +rank0 [2026-11-15 04:38:25,640] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:38:25,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:25,311] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:39:25,311] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:01,572] INFO : Step 29000/500000 rank0 | Train Loss: 10.0288 | Train Loss Unweighted: 2.3887 | Train Loss Components: Classification: 0.0112 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0700 | Regression_eta: 0.0042 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3873 | Classification_binary: 2.2258 | Regression_cos_phi: 0.9708 | Regression_energy: 1.4173 | Regression_eta: 1.3666 | Regression_pt: 1.4433 | Regression_sin_phi: 1.2178 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3482.5 | uss_MB: 2256.0 | child_rss_MB: 22813.3 | allocated_MB: 170.4 | reserved_MB: 700.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:01,574] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:01,574] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:01,574] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:01,719] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:40:01,837] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:01,949] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:02,018] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:02,085] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:02,152] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:02,218] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:02,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:02,392] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:02,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:02,622] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:02,690] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:02,758] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:02,825] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:02,892] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:02,959] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:03,074] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:03,081] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:03,084] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:03,084] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:03,088] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:03,090] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:03,092] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:03,094] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:03,096] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:03,098] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:03,100] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:03,102] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:03,104] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:03,105] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:03,107] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:03,109] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:04,551] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.0288 | Valid Loss=15.7530 | Train Loss Unweighted=2.3887 | Valid Loss Unweighted=2.3303 | Stale=24 | ETA=928.3m +rank0 [2026-11-15 04:40:04,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:04,628] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:04,631] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:04,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:05,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:05,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:05,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:05,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:06,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:06,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:06,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:06,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:07,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:07,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:07,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:07,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:08,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:08,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:08,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:08,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:08,978] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:40:08,984] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,562] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,948] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:11,253] INFO : Plotted jets +rank0 [2026-11-15 04:40:12,670] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:36,798] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:40:36,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:36,508] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:41:36,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:59,672] INFO : Step 30000/500000 rank0 | Train Loss: 8.4215 | Train Loss Unweighted: 1.9282 | Train Loss Components: Classification: 0.0135 | Classification_binary: 1.7875 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0608 | Regression_eta: 0.0022 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6765 | Classification_binary: 1.7875 | Regression_cos_phi: 0.8244 | Regression_energy: 1.2305 | Regression_eta: 0.7078 | Regression_pt: 1.1963 | Regression_sin_phi: 0.9985 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3506.2 | uss_MB: 2279.8 | child_rss_MB: 22814.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:59,673] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:41:59,740] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:59,740] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:59,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:59,891] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:42:00,009] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:00,128] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:00,196] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:00,263] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:00,330] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:00,398] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:00,464] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:00,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:00,682] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:00,795] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:00,861] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:00,925] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:00,989] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:01,055] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:01,121] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:01,234] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:01,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:01,243] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:01,244] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:01,248] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:01,248] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:01,252] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:01,254] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:01,256] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:01,256] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:01,260] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:01,262] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:01,264] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:01,266] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:01,266] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:01,269] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:02,720] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.4215 | Valid Loss=15.7263 | Train Loss Unweighted=1.9282 | Valid Loss Unweighted=2.3297 | Stale=25 | ETA=926.3m +rank0 [2026-11-15 04:42:02,721] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:02,797] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:02,802] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:02,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:03,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:03,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:03,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:04,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:04,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:04,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:04,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:05,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:05,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:05,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:05,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:06,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:06,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:06,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:06,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:07,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:07,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:42:07,144] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:07,709] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:08,091] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:09,406] INFO : Plotted jets +rank0 [2026-11-15 04:42:10,833] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:48,190] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:42:48,190] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:47,962] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:43:47,962] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:57,912] INFO : Step 31000/500000 rank0 | Train Loss: 9.0567 | Train Loss Unweighted: 2.3107 | Train Loss Components: Classification: 0.0113 | Classification_binary: 2.1404 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0746 | Regression_eta: 0.0027 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3955 | Classification_binary: 2.1404 | Regression_cos_phi: 0.8884 | Regression_energy: 1.5099 | Regression_eta: 0.8848 | Regression_pt: 1.5591 | Regression_sin_phi: 0.6786 | LR: 6.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4851s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3446.3 | uss_MB: 2219.8 | child_rss_MB: 22809.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:43:57,913] INFO : Running validation on rank0 +rank0 [2026-11-15 04:43:57,913] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:43:57,913] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:43:58,062] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:43:58,182] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:43:58,295] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:43:58,364] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:43:58,429] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:43:58,493] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:43:58,558] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:43:58,622] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:43:58,726] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:43:58,840] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:43:58,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:43:59,020] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:43:59,089] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:43:59,156] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:43:59,223] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:43:59,289] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:43:59,401] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:43:59,408] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:43:59,409] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:43:59,412] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:43:59,414] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:43:59,416] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:43:59,417] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:43:59,419] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:43:59,421] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:43:59,423] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:43:59,426] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:43:59,427] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:43:59,429] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:43:59,431] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:43:59,433] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:43:59,437] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:00,883] INFO : VALIDATION | Step=31000/500000 | Train Loss=9.0567 | Valid Loss=15.7236 | Train Loss Unweighted=2.3107 | Valid Loss Unweighted=2.3357 | Stale=26 | ETA=924.3m +rank0 [2026-11-15 04:44:00,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:00,954] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:00,958] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:01,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:01,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:01,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:02,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:02,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:02,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:02,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:03,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:03,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:03,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:03,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:04,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:04,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:04,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:04,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:05,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:05,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:05,324] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:44:05,330] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:05,912] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:06,304] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:07,594] INFO : Plotted jets +rank0 [2026-11-15 04:44:09,007] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:59,524] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:44:59,524] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:56,126] INFO : Step 32000/500000 rank0 | Train Loss: 10.7088 | Train Loss Unweighted: 2.0974 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9360 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0685 | Regression_eta: 0.0034 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6150 | Classification_binary: 1.9360 | Regression_cos_phi: 1.8698 | Regression_energy: 1.3862 | Regression_eta: 1.0838 | Regression_pt: 1.3663 | Regression_sin_phi: 1.4517 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4796s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1235 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3555.4 | uss_MB: 2329.0 | child_rss_MB: 22806.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:45:56,128] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:45:56,194] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:56,194] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:56,194] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:56,336] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:45:56,454] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:56,567] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:56,636] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:56,702] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:56,767] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:56,834] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:56,900] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:57,006] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:57,125] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:57,240] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:57,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:57,374] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:57,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:57,503] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:57,568] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:57,677] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:57,684] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:57,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:57,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:57,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:57,691] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:57,695] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:57,695] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:57,697] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:57,701] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:57,702] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:57,703] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:57,706] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:57,708] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:57,709] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:57,713] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:59,159] INFO : VALIDATION | Step=32000/500000 | Train Loss=10.7088 | Valid Loss=15.9452 | Train Loss Unweighted=2.0974 | Valid Loss Unweighted=2.3487 | Stale=27 | ETA=922.3m +rank0 [2026-11-15 04:45:59,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:59,233] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:59,236] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:59,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:59,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:00,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:00,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:00,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:00,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:01,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:01,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:01,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:01,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:02,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:02,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:02,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:02,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:03,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:03,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:03,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:03,655] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:46:03,661] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:04,235] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:04,613] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:05,914] INFO : Plotted jets +rank0 [2026-11-15 04:46:07,312] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:11,268] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:46:11,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:11,222] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:47:11,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:54,753] INFO : Step 33000/500000 rank0 | Train Loss: 6.8837 | Train Loss Unweighted: 1.9010 | Train Loss Components: Classification: 0.0075 | Classification_binary: 1.7671 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0599 | Regression_eta: 0.0016 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 0.9281 | Classification_binary: 1.7671 | Regression_cos_phi: 0.5688 | Regression_energy: 1.2130 | Regression_eta: 0.5010 | Regression_pt: 1.2401 | Regression_sin_phi: 0.6656 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5037s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0649 | max_tensorboard_logging: 0.1399 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3588.3 | uss_MB: 2426.1 | child_rss_MB: 22807.1 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:47:54,754] INFO : Running validation on rank0 +rank0 [2026-11-15 04:47:54,754] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:47:54,754] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:47:54,900] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:47:55,018] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:47:55,135] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:47:55,203] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:47:55,269] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:47:55,334] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:47:55,401] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:47:55,468] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:47:55,576] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:47:55,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:47:55,808] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:47:55,876] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:47:55,943] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:47:56,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:47:56,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:47:56,149] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:47:56,262] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:47:56,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:47:56,270] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:47:56,280] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:47:56,280] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:47:56,282] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:47:56,285] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:47:56,286] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:47:56,290] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:47:56,291] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:47:56,294] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:47:56,295] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:47:56,298] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:47:56,300] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:47:56,301] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:47:56,304] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:57,772] INFO : VALIDATION | Step=33000/500000 | Train Loss=6.8837 | Valid Loss=15.9740 | Train Loss Unweighted=1.9010 | Valid Loss Unweighted=2.3558 | Stale=28 | ETA=920.4m +rank0 [2026-11-15 04:47:57,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:57,859] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:47:57,862] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:57,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:58,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:58,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:58,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:59,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:59,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:59,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:59,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:00,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:00,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:00,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:00,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:01,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:01,484] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:01,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:01,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:02,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:02,270] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:48:02,276] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:02,851] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:03,240] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:04,553] INFO : Plotted jets +rank0 [2026-11-15 04:48:05,977] INFO : Plotted jet ratio +rank0 [2026-11-15 04:48:23,295] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 04:48:23,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:23,216] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 04:49:23,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:53,502] INFO : Step 34000/500000 rank0 | Train Loss: 9.4840 | Train Loss Unweighted: 2.1458 | Train Loss Components: Classification: 0.0107 | Classification_binary: 1.9642 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0824 | Regression_eta: 0.0037 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3231 | Classification_binary: 1.9642 | Regression_cos_phi: 0.8855 | Regression_energy: 1.6677 | Regression_eta: 1.1855 | Regression_pt: 1.6159 | Regression_sin_phi: 0.8421 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0396 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1532 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3596.6 | uss_MB: 2434.4 | child_rss_MB: 22803.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:49:53,504] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:49:53,570] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:53,570] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:53,570] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:53,713] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:49:53,832] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:53,944] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:54,013] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:54,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:54,144] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:54,210] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:54,276] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:54,381] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:54,497] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:54,610] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:54,678] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:54,745] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:54,811] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:54,878] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:54,945] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:55,056] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:55,063] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:55,066] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:55,068] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:55,070] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:55,072] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:55,074] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:55,075] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:55,078] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:55,081] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:55,083] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:55,083] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:55,085] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:55,089] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:55,090] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:55,092] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:56,535] INFO : VALIDATION | Step=34000/500000 | Train Loss=9.4840 | Valid Loss=16.2660 | Train Loss Unweighted=2.1458 | Valid Loss Unweighted=2.3731 | Stale=29 | ETA=918.6m +rank0 [2026-11-15 04:49:56,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:56,621] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:56,629] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:56,677] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:57,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:57,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:57,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:57,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:58,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:58,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:58,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:58,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:59,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:59,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:59,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:59,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:50:00,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:50:00,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:50:00,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:50:00,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:50:00,987] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:50:00,993] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:50:01,578] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:50:01,958] INFO : Plotted number of elements +rank0 [2026-11-15 04:50:03,265] INFO : Plotted jets +rank0 [2026-11-15 04:50:04,685] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:35,216] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 04:50:35,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:35,184] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 04:51:35,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:52,278] INFO : Step 35000/500000 rank0 | Train Loss: 10.8430 | Train Loss Unweighted: 2.0802 | Train Loss Components: Classification: 0.0122 | Classification_binary: 1.8954 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0795 | Regression_eta: 0.0069 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5136 | Classification_binary: 1.8954 | Regression_cos_phi: 0.8530 | Regression_energy: 1.6094 | Regression_eta: 2.2143 | Regression_pt: 1.6295 | Regression_sin_phi: 1.1278 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5405s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1735 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3640.2 | uss_MB: 2477.9 | child_rss_MB: 22812.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 19.6 +rank0 [2026-11-15 04:51:52,280] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:52,280] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:52,280] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:52,423] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 04:51:52,539] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:52,653] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:52,721] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:52,787] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:52,852] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:52,919] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:52,987] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:53,095] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:53,211] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:53,327] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:53,396] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:53,461] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:53,525] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:53,590] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:53,655] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:53,764] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:53,770] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:53,772] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:53,776] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:53,778] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:53,778] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:53,781] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:53,782] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:53,784] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:53,788] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:53,788] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:53,792] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:53,794] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:53,796] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:53,798] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:53,800] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:55,253] INFO : VALIDATION | Step=35000/500000 | Train Loss=10.8430 | Valid Loss=16.2588 | Train Loss Unweighted=2.0802 | Valid Loss Unweighted=2.3791 | Stale=30 | ETA=916.7m +rank0 [2026-11-15 04:51:55,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:55,350] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:55,358] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:55,406] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:55,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:56,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:56,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:56,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:56,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:57,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:57,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:57,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:57,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:58,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:58,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:58,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:58,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:59,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:59,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:59,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:59,749] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:51:59,755] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:00,335] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:00,722] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:02,041] INFO : Plotted jets +rank0 [2026-11-15 04:52:03,487] INFO : Plotted jet ratio +rank0 [2026-11-15 04:52:03,837] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:52:03,837] INFO : Training completed. Total time on device 0: 69.143min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..1bacd21e1b4959281f66f79db2f3f686320d862f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/train.log.0 @@ -0,0 +1,2763 @@ +rank0 [2026-11-15 03:42:53,119] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:42:53,119] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:42:53,120] INFO : Instantiating model +rank0 [2026-11-15 03:42:53,142] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:42:53,143] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:42:53,144] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:42:53,144] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:42:53,144] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:42:53,144] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:42:53,144] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,147] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:42:53,147] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,150] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:42:53,150] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,152] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:42:53,152] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,155] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:42:53,155] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,157] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:42:53,158] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,160] INFO : Backbone initialization took 0.02s +rank0 [2026-11-15 03:42:53,160] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:42:53,160] INFO : Initializing output DNNs +rank0 [2026-11-15 03:42:53,161] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,164] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:42:53,171] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:42:53,171] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:42:53,171] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:42:53,171] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:42:53,171] INFO : task_queries enabled=True +rank0 [2026-11-15 03:42:53,171] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:42:53,171] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:42:53,171] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:42:53,171] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:42:53,171] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:42:53,171] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:42:53,171] INFO : output_mode=elementwise +rank0 [2026-11-15 03:42:53,171] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:42:53,172] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:42:53,172] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:42:53,172] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:42:53,172] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:42:53,173] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:42:53,173] INFO : MLPF __init__ done +rank0 [2026-11-15 03:42:53,173] INFO : Instantiated model +rank0 [2026-11-15 03:42:53,174] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:42:53,182] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:42:53,183] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:42:53,305] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:42:53,306] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:42:53,306] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:42:53,399] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:42:53,399] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:42:53,401] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:42:53,401] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:42:53,401] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:42:53,403] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:42:53,408] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046 +rank0 [2026-11-15 03:42:53,408] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046 +rank0 [2026-11-15 03:42:53,408] INFO : Getting dataloaders +rank0 [2026-11-15 03:42:53,408] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:54,612] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,659] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,702] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,744] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,781] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,812] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,852] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,882] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,938] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,982] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:42:54,983] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:42:54,983] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:42:54,983] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:42:54,983] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:42:54,990] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:54,997] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,004] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,012] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,019] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,026] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,033] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,040] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,047] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,054] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:42:55,054] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:42:55,054] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:42:55,101] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:55,234] INFO : Got dataloaders +rank0 [2026-11-15 03:42:55,234] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:42:55,234] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:42:55,234] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:42:55,243] INFO : Creating train_iterator +rank0 [2026-11-15 03:42:55,244] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:42:55,268] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:42:55,393] INFO : Created train_iterator +rank0 [2026-11-15 03:43:07,394] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9870172393321992, 'Classification': 0.016035858988761902, 'Regression_pt': 0.09887309789657593, 'Regression_eta': 0.006165142094250769, 'Regression_sin_phi': 0.005006907158531249, 'Regression_cos_phi': 0.0051970563700888305, 'Regression_energy': 0.09819029666483402}, weights={'Classification_binary': 1.0, 'Classification': 123.91087341308594, 'Regression_pt': 20.096641540527344, 'Regression_eta': 322.2987060546875, 'Regression_sin_phi': 396.855224609375, 'Regression_cos_phi': 382.33514404296875, 'Regression_energy': 20.236391067504883} +rank0 [2026-11-15 03:43:57,269] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:43:57,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:44:44,189] INFO : Step 1000/500000 rank0 | Train Loss: 11.6175 | Train Loss Unweighted: 2.1554 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9803 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0730 | Regression_eta: 0.0054 | Regression_pt: 0.0753 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6093 | Classification_binary: 1.9803 | Regression_cos_phi: 1.4906 | Regression_energy: 1.4782 | Regression_eta: 1.7275 | Regression_pt: 1.5126 | Regression_sin_phi: 1.8190 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4618s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0024 | max_backward: 0.0404 | max_optimizer_step: 0.0567 | max_tensorboard_logging: 0.1322 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2609.0 | uss_MB: 1373.4 | child_rss_MB: 22794.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 03:44:44,190] INFO : Running validation on rank0 +rank0 [2026-11-15 03:44:44,190] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:44:44,190] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:44:44,373] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:44:44,555] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:44:44,635] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:44:44,689] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:44:44,743] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:44:44,790] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:44:44,844] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:44:44,900] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:44:44,949] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:44:45,025] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:44:45,104] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:44:45,156] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:44:45,210] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:44:45,254] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:44:45,307] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:44:45,360] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:44:45,409] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:44:45,413] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:44:45,415] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:44:45,418] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:44:45,419] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:44:45,421] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:44:45,422] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:44:45,425] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:44:45,427] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:44:45,429] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:44:45,430] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:44:45,433] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:44:45,435] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:44:45,436] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:44:45,438] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:44:45,441] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:44:46,948] INFO : VALIDATION | Step=1000/500000 | Train Loss=11.6175 | Valid Loss=13.8300 | Train Loss Unweighted=2.1554 | Valid Loss Unweighted=2.2064 | Stale=0 | ETA=929.0m +rank0 [2026-11-15 03:44:46,948] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:44:47,022] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:44:47,029] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:47,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:44:47,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:44:47,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:44:48,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:44:48,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:44:48,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:44:48,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:44:49,244] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:44:49,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:44:49,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:44:50,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:44:50,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:44:50,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:44:50,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:44:50,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:44:51,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:44:51,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:44:51,533] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:44:51,541] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:52,103] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:44:52,820] INFO : Plotted number of elements +rank0 [2026-11-15 03:44:54,177] INFO : Plotted jets +rank0 [2026-11-15 03:44:55,543] INFO : Plotted jet ratio +rank0 [2026-11-15 03:45:09,059] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:45:09,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:08,518] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:46:08,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:46:41,981] INFO : Step 2000/500000 rank0 | Train Loss: 11.2152 | Train Loss Unweighted: 1.8050 | Train Loss Components: Classification: 0.0138 | Classification_binary: 1.6116 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0839 | Regression_eta: 0.0031 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.7091 | Classification_binary: 1.6116 | Regression_cos_phi: 2.0506 | Regression_energy: 1.6977 | Regression_eta: 0.9979 | Regression_pt: 1.6789 | Regression_sin_phi: 1.4693 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4593s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0394 | max_optimizer_step: 0.0591 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.5 | uss_MB: 1553.7 | child_rss_MB: 22797.3 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 49.6 +rank0 [2026-11-15 03:46:41,983] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:46:42,052] INFO : Running validation on rank0 +rank0 [2026-11-15 03:46:42,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:46:42,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:46:42,212] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:46:42,331] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:46:42,445] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:46:42,512] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:46:42,576] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:46:42,639] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:46:42,703] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:46:42,768] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:46:42,872] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:46:42,984] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:46:43,097] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:46:43,164] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:46:43,229] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:46:43,292] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:46:43,356] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:46:43,420] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:46:43,532] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:46:43,539] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:46:43,541] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:46:43,542] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:46:43,545] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:46:43,547] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:46:43,548] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:46:43,551] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:46:43,553] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:46:43,555] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:46:43,557] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:46:43,559] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:46:43,561] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:46:43,563] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:46:43,565] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:46:43,567] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:46:45,017] INFO : VALIDATION | Step=2000/500000 | Train Loss=11.2152 | Valid Loss=13.8985 | Train Loss Unweighted=1.8050 | Valid Loss Unweighted=2.2112 | Stale=1 | ETA=953.6m +rank0 [2026-11-15 03:46:45,018] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:46:45,090] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:46:45,096] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:45,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:46:45,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:46:45,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:46:46,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:46:46,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:46:46,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:46:46,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:46:47,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:46:47,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:46:47,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:46:47,829] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:46:48,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:46:48,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:46:48,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:46:48,832] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:46:49,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:46:49,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:46:49,356] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:46:49,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:49,923] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:46:50,313] INFO : Plotted number of elements +rank0 [2026-11-15 03:46:52,007] INFO : Plotted jets +rank0 [2026-11-15 03:46:52,990] INFO : Plotted jet ratio +rank0 [2026-11-15 03:47:19,537] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:47:19,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:19,142] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:48:19,142] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:48:39,659] INFO : Step 3000/500000 rank0 | Train Loss: 15.6563 | Train Loss Unweighted: 2.1858 | Train Loss Components: Classification: 0.0150 | Classification_binary: 1.9451 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1012 | Regression_eta: 0.0088 | Regression_pt: 0.1032 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.8567 | Classification_binary: 1.9451 | Regression_cos_phi: 1.8755 | Regression_energy: 2.0471 | Regression_eta: 2.8304 | Regression_pt: 2.0734 | Regression_sin_phi: 3.0282 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4662s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0595 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2834.2 | uss_MB: 1607.6 | child_rss_MB: 22796.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 03:48:39,660] INFO : Running validation on rank0 +rank0 [2026-11-15 03:48:39,660] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:48:39,660] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:48:39,806] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:48:39,931] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:48:40,046] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:48:40,115] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:48:40,181] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:48:40,246] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:48:40,313] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:48:40,379] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:48:40,484] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:48:40,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:48:40,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:48:40,772] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:48:40,837] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:48:40,901] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:48:40,966] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:48:41,031] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:48:41,143] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:48:41,150] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:48:41,152] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:48:41,153] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:48:41,155] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:48:41,157] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:48:41,159] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:48:41,161] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:48:41,163] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:48:41,165] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:48:41,168] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:48:41,170] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:48:41,172] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:48:41,174] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:48:41,176] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:48:41,177] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:48:42,620] INFO : VALIDATION | Step=3000/500000 | Train Loss=15.6563 | Valid Loss=13.8469 | Train Loss Unweighted=2.1858 | Valid Loss Unweighted=2.2132 | Stale=2 | ETA=959.1m +rank0 [2026-11-15 03:48:42,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:48:42,696] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:48:42,712] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:42,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:48:43,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:48:43,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:48:43,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:48:43,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:48:44,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:48:44,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:48:44,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:48:44,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:48:45,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:48:45,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:48:45,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:48:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:48:46,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:48:46,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:48:46,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:48:46,932] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:48:46,986] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:48:46,991] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:47,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:48:47,927] INFO : Plotted number of elements +rank0 [2026-11-15 03:48:49,634] INFO : Plotted jets +rank0 [2026-11-15 03:48:50,609] INFO : Plotted jet ratio +rank0 [2026-11-15 03:49:30,553] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:49:30,554] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:30,429] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 03:50:30,429] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:50:37,810] INFO : Step 4000/500000 rank0 | Train Loss: 14.9054 | Train Loss Unweighted: 2.1119 | Train Loss Components: Classification: 0.0197 | Classification_binary: 1.9076 | Regression_cos_phi: 0.0097 | Regression_energy: 0.0844 | Regression_eta: 0.0037 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.4465 | Classification_binary: 1.9076 | Regression_cos_phi: 3.6956 | Regression_energy: 1.7070 | Regression_eta: 1.1852 | Regression_pt: 1.6267 | Regression_sin_phi: 2.3368 | LR: 2.08e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4510s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0344 | max_loss: 0.0024 | max_backward: 0.0386 | max_optimizer_step: 0.0587 | max_tensorboard_logging: 0.1178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2866.4 | uss_MB: 1639.8 | child_rss_MB: 22801.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:50:37,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 03:50:37,876] INFO : Running validation on rank0 +rank0 [2026-11-15 03:50:37,876] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:50:37,876] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:50:38,020] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 03:50:38,136] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:50:38,251] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:50:38,319] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:50:38,385] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:50:38,450] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:50:38,517] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,584] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:50:38,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:50:38,803] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:50:38,912] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:50:38,978] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:50:39,043] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:50:39,107] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:50:39,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,236] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:50:39,347] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:50:39,354] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:50:39,356] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:50:39,359] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:50:39,361] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:50:39,361] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:50:39,363] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:50:39,366] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:50:39,367] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:50:39,369] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:50:39,371] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:50:39,374] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:50:39,375] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:50:39,378] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:50:39,380] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:50:39,381] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:50:40,884] INFO : VALIDATION | Step=4000/500000 | Train Loss=14.9054 | Valid Loss=13.8275 | Train Loss Unweighted=2.1119 | Valid Loss Unweighted=2.2141 | Stale=0 | ETA=962.3m +rank0 [2026-11-15 03:50:40,884] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:50:40,957] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:50:40,963] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:41,013] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:50:41,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:50:41,774] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:50:42,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:50:42,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:50:42,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:50:42,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:50:43,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:50:43,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:50:43,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:50:43,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:50:44,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:50:44,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:50:44,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:50:44,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:50:45,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:50:45,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:50:45,289] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:50:45,294] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:45,853] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:50:46,243] INFO : Plotted number of elements +rank0 [2026-11-15 03:50:47,942] INFO : Plotted jets +rank0 [2026-11-15 03:50:48,913] INFO : Plotted jet ratio +rank0 [2026-11-15 03:51:41,957] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 03:51:41,957] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:52:35,779] INFO : Step 5000/500000 rank0 | Train Loss: 12.6854 | Train Loss Unweighted: 2.1025 | Train Loss Components: Classification: 0.0124 | Classification_binary: 1.8557 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1073 | Regression_eta: 0.0038 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5405 | Classification_binary: 1.8557 | Regression_cos_phi: 2.0577 | Regression_energy: 2.1720 | Regression_eta: 1.2281 | Regression_pt: 2.2905 | Regression_sin_phi: 1.5410 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0593 | max_tensorboard_logging: 0.1260 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2885.3 | uss_MB: 1658.8 | child_rss_MB: 22802.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:52:35,780] INFO : Running validation on rank0 +rank0 [2026-11-15 03:52:35,780] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:52:35,780] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:52:35,926] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 03:52:36,041] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:52:36,157] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:52:36,225] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:52:36,290] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:52:36,356] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:52:36,423] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:52:36,490] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:52:36,595] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:52:36,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:52:36,821] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:52:36,887] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:52:36,952] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:52:37,015] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:52:37,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:52:37,147] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:52:37,256] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:52:37,262] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:52:37,265] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:52:37,267] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:52:37,269] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:52:37,271] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:52:37,273] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:52:37,275] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:52:37,277] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:52:37,279] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:52:37,281] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:52:37,283] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:52:37,285] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:52:37,287] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:52:37,289] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:52:37,291] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:52:38,795] INFO : VALIDATION | Step=5000/500000 | Train Loss=12.6854 | Valid Loss=13.8205 | Train Loss Unweighted=2.1025 | Valid Loss Unweighted=2.2168 | Stale=0 | ETA=962.9m +rank0 [2026-11-15 03:52:38,795] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:52:38,870] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:52:38,876] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:38,925] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:52:39,421] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:52:39,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:52:39,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:52:40,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:52:40,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:52:40,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:52:40,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:52:41,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:52:41,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:52:41,677] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:52:41,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:52:42,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:52:42,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:52:42,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:52:42,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:52:43,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:52:43,194] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:52:43,201] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:43,764] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:52:44,146] INFO : Plotted number of elements +rank0 [2026-11-15 03:52:45,451] INFO : Plotted jets +rank0 [2026-11-15 03:52:46,833] INFO : Plotted jet ratio +rank0 [2026-11-15 03:52:53,337] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 03:52:53,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:53:53,322] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 03:53:53,322] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:54:34,362] INFO : Step 6000/500000 rank0 | Train Loss: 9.6100 | Train Loss Unweighted: 2.0251 | Train Loss Components: Classification: 0.0145 | Classification_binary: 1.8540 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0737 | Regression_eta: 0.0025 | Regression_pt: 0.0750 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.7945 | Classification_binary: 1.8540 | Regression_cos_phi: 0.8898 | Regression_energy: 1.4906 | Regression_eta: 0.7922 | Regression_pt: 1.5066 | Regression_sin_phi: 1.2822 | LR: 2.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4704s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1279 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.6 | uss_MB: 1603.1 | child_rss_MB: 22806.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 03:54:34,364] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 03:54:34,431] INFO : Running validation on rank0 +rank0 [2026-11-15 03:54:34,431] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:54:34,431] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:54:34,575] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 03:54:34,694] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:54:34,806] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:54:34,875] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:54:34,940] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:54:35,005] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:54:35,073] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:54:35,140] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:54:35,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:54:35,360] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:54:35,474] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:54:35,541] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:54:35,607] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:54:35,672] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:54:35,737] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:54:35,802] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:54:35,911] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:54:35,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:54:35,920] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:54:35,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:54:35,924] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:54:35,926] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:54:35,927] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:54:35,930] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:54:35,932] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:54:35,934] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:54:35,936] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:54:35,938] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:54:35,939] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:54:35,942] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:54:35,944] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:54:35,946] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:54:37,391] INFO : VALIDATION | Step=6000/500000 | Train Loss=9.6100 | Valid Loss=13.8499 | Train Loss Unweighted=2.0251 | Valid Loss Unweighted=2.2220 | Stale=1 | ETA=963.5m +rank0 [2026-11-15 03:54:37,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:37,465] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:54:37,472] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:37,521] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:38,039] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:54:38,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:54:38,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:54:38,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:54:39,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:54:39,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:54:39,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:54:39,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:54:40,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:54:40,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:54:40,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:54:40,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:54:41,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:54:41,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:54:41,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:54:41,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:54:41,791] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:54:41,796] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:42,364] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:54:42,741] INFO : Plotted number of elements +rank0 [2026-11-15 03:54:44,435] INFO : Plotted jets +rank0 [2026-11-15 03:54:45,408] INFO : Plotted jet ratio +rank0 [2026-11-15 03:55:05,004] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 03:55:05,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:56:05,242] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 03:56:05,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:56:32,915] INFO : Step 7000/500000 rank0 | Train Loss: 14.5605 | Train Loss Unweighted: 1.9631 | Train Loss Components: Classification: 0.0178 | Classification_binary: 1.7586 | Regression_cos_phi: 0.0078 | Regression_energy: 0.0815 | Regression_eta: 0.0051 | Regression_pt: 0.0858 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.2046 | Classification_binary: 1.7586 | Regression_cos_phi: 2.9650 | Regression_energy: 1.6496 | Regression_eta: 1.6587 | Regression_pt: 1.7235 | Regression_sin_phi: 2.6006 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4681s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1262 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2940.6 | uss_MB: 1714.2 | child_rss_MB: 22804.1 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:56:32,916] INFO : Running validation on rank0 +rank0 [2026-11-15 03:56:32,916] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:56:32,916] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:56:33,064] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 03:56:33,181] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:56:33,293] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:56:33,361] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:56:33,426] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:56:33,491] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:56:33,557] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:56:33,622] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:56:33,728] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:56:33,844] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:56:33,958] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:56:34,026] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:56:34,094] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:56:34,160] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:56:34,225] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:56:34,289] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:56:34,397] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:56:34,402] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:56:34,404] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:56:34,407] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:56:34,409] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:56:34,410] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:56:34,414] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:56:34,416] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:56:34,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:56:34,420] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:56:34,422] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:56:34,422] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:56:34,426] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:56:34,426] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:56:34,428] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:56:34,430] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:56:35,870] INFO : VALIDATION | Step=7000/500000 | Train Loss=14.5605 | Valid Loss=13.8935 | Train Loss Unweighted=1.9631 | Valid Loss Unweighted=2.2243 | Stale=2 | ETA=963.3m +rank0 [2026-11-15 03:56:35,870] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:56:35,944] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:56:35,950] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:35,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:56:36,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:56:36,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:56:36,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:56:37,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:56:37,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:56:37,738] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:56:37,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:56:38,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:56:38,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:56:38,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:56:39,008] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:56:39,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:56:39,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:56:39,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:56:39,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:56:40,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:56:40,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:56:40,304] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:40,871] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:41,252] INFO : Plotted number of elements +rank0 [2026-11-15 03:56:42,959] INFO : Plotted jets +rank0 [2026-11-15 03:56:43,929] INFO : Plotted jet ratio +rank0 [2026-11-15 03:57:16,810] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 03:57:16,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:16,899] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 03:58:16,899] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:31,349] INFO : Step 8000/500000 rank0 | Train Loss: 13.5738 | Train Loss Unweighted: 2.1991 | Train Loss Components: Classification: 0.0137 | Classification_binary: 1.9968 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0851 | Regression_eta: 0.0063 | Regression_pt: 0.0859 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6962 | Classification_binary: 1.9968 | Regression_cos_phi: 1.8861 | Regression_energy: 1.7217 | Regression_eta: 2.0379 | Regression_pt: 1.7254 | Regression_sin_phi: 2.5097 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4754s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0427 | max_optimizer_step: 0.0611 | max_tensorboard_logging: 0.1204 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2963.3 | uss_MB: 1736.8 | child_rss_MB: 22804.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:58:31,351] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 03:58:31,416] INFO : Running validation on rank0 +rank0 [2026-11-15 03:58:31,416] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:58:31,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:58:31,559] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 03:58:31,677] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:58:31,788] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:58:31,855] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:58:31,920] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:58:31,984] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:58:32,051] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:58:32,117] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:58:32,222] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:58:32,336] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:58:32,451] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:58:32,517] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:58:32,583] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:58:32,649] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:58:32,715] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:58:32,782] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:58:32,891] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:58:32,898] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:58:32,898] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:58:32,901] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:58:32,904] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:58:32,906] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:58:32,907] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:58:32,910] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:58:32,912] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:58:32,914] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:58:32,916] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:58:32,918] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:58:32,920] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:58:32,922] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:58:32,924] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:58:32,926] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:58:34,379] INFO : VALIDATION | Step=8000/500000 | Train Loss=13.5738 | Valid Loss=13.8437 | Train Loss Unweighted=2.1991 | Valid Loss Unweighted=2.2264 | Stale=3 | ETA=962.6m +rank0 [2026-11-15 03:58:34,379] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:58:34,451] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:58:34,457] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:34,506] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:58:34,994] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:58:35,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:58:35,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:58:35,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:58:36,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:58:36,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:58:36,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:58:36,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:58:37,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:58:37,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:58:37,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:58:37,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:58:38,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:58:38,253] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:58:38,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:58:38,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:58:38,774] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 03:58:38,782] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:39,357] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:39,737] INFO : Plotted number of elements +rank0 [2026-11-15 03:58:41,448] INFO : Plotted jets +rank0 [2026-11-15 03:58:42,431] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:28,658] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 03:59:28,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:28,693] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:00:28,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:29,936] INFO : Step 9000/500000 rank0 | Train Loss: 12.3439 | Train Loss Unweighted: 2.1709 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.9652 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0892 | Regression_eta: 0.0034 | Regression_pt: 0.0898 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5895 | Classification_binary: 1.9652 | Regression_cos_phi: 1.9989 | Regression_energy: 1.8048 | Regression_eta: 1.0958 | Regression_pt: 1.8040 | Regression_sin_phi: 2.0856 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4640s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1265 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2903.4 | uss_MB: 1677.0 | child_rss_MB: 22804.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:00:29,937] INFO : Running validation on rank0 +rank0 [2026-11-15 04:00:29,937] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:00:29,937] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:00:30,086] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:00:30,206] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:00:30,319] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:00:30,388] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:00:30,452] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:00:30,517] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:00:30,583] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:00:30,649] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:00:30,756] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:00:30,872] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:00:30,987] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:00:31,056] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:00:31,123] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:00:31,188] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:00:31,254] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:00:31,320] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:00:31,430] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:00:31,437] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:00:31,439] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:00:31,441] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:00:31,443] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:00:31,445] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:00:31,446] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:00:31,448] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:00:31,450] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:00:31,452] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:00:31,455] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:00:31,456] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:00:31,458] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:00:31,462] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:00:31,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:00:31,466] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:32,917] INFO : VALIDATION | Step=9000/500000 | Train Loss=12.3439 | Valid Loss=13.8528 | Train Loss Unweighted=2.1709 | Valid Loss Unweighted=2.2233 | Stale=4 | ETA=961.7m +rank0 [2026-11-15 04:00:32,917] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:00:32,991] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:00:32,999] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:33,049] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:33,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:33,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:34,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:34,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:34,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:34,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:35,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:35,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:35,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:35,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:36,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:36,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:36,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:36,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:37,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:37,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:37,288] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:00:37,294] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:37,863] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:38,244] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:39,941] INFO : Plotted jets +rank0 [2026-11-15 04:00:40,917] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:40,321] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:01:40,321] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:28,457] INFO : Step 10000/500000 rank0 | Train Loss: 12.3633 | Train Loss Unweighted: 2.1192 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.9369 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0784 | Regression_eta: 0.0052 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4968 | Classification_binary: 1.9369 | Regression_cos_phi: 2.1570 | Regression_energy: 1.5867 | Regression_eta: 1.6890 | Regression_pt: 1.5279 | Regression_sin_phi: 1.9690 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4852s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0026 | max_backward: 0.0394 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.9 | uss_MB: 1785.4 | child_rss_MB: 22804.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:02:28,459] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:02:28,525] INFO : Running validation on rank0 +rank0 [2026-11-15 04:02:28,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:02:28,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:02:28,669] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:02:28,787] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:02:28,902] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:02:28,969] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:02:29,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:02:29,103] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:02:29,170] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:02:29,237] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:02:29,344] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:02:29,459] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:02:29,574] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:02:29,642] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:02:29,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:02:29,773] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:02:29,839] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:02:29,904] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:02:30,013] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:02:30,020] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:02:30,022] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:02:30,023] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:02:30,027] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:02:30,027] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:02:30,030] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:02:30,033] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:02:30,034] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:02:30,037] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:02:30,039] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:02:30,041] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:02:30,043] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:02:30,045] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:02:30,047] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:02:30,050] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:02:31,515] INFO : VALIDATION | Step=10000/500000 | Train Loss=12.3633 | Valid Loss=13.9266 | Train Loss Unweighted=2.1192 | Valid Loss Unweighted=2.2295 | Stale=5 | ETA=960.6m +rank0 [2026-11-15 04:02:31,515] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:02:31,590] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:02:31,594] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:31,644] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:02:32,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:02:32,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:02:32,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:02:32,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:02:33,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:02:33,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:02:33,650] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:02:33,912] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:02:34,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:02:34,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:02:34,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:02:34,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:02:35,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:02:35,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:02:35,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:02:35,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:02:35,960] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:02:35,968] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:36,533] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:36,912] INFO : Plotted number of elements +rank0 [2026-11-15 04:02:38,639] INFO : Plotted jets +rank0 [2026-11-15 04:02:39,630] INFO : Plotted jet ratio +rank0 [2026-11-15 04:02:52,344] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:02:52,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:03:52,438] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:03:52,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:27,174] INFO : Step 11000/500000 rank0 | Train Loss: 12.0314 | Train Loss Unweighted: 2.1470 | Train Loss Components: Classification: 0.0153 | Classification_binary: 1.9321 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0918 | Regression_eta: 0.0034 | Regression_pt: 0.0959 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.8947 | Classification_binary: 1.9321 | Regression_cos_phi: 1.5363 | Regression_energy: 1.8579 | Regression_eta: 1.0929 | Regression_pt: 1.9270 | Regression_sin_phi: 1.7905 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0603 | max_tensorboard_logging: 0.1249 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3037.0 | uss_MB: 1810.6 | child_rss_MB: 22811.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:04:27,175] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:27,175] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:27,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:27,320] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:04:27,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:27,549] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:27,617] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:27,682] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:27,747] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:27,812] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:27,878] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:27,984] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:28,102] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:28,217] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:28,285] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:28,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:28,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:28,482] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:28,547] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:28,658] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:28,664] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:28,667] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:28,668] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:28,671] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:28,673] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:28,676] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:28,677] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:28,679] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:28,681] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:28,683] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:28,684] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:28,687] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:28,688] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:28,690] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:28,694] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:30,155] INFO : VALIDATION | Step=11000/500000 | Train Loss=12.0314 | Valid Loss=14.0317 | Train Loss Unweighted=2.1470 | Valid Loss Unweighted=2.2364 | Stale=6 | ETA=959.4m +rank0 [2026-11-15 04:04:30,156] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:04:30,231] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:04:30,237] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:30,287] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:30,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:31,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:31,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:31,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:31,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:32,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:32,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:32,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:32,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:33,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:33,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:33,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:33,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:34,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:34,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:34,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:34,582] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:04:34,600] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:35,168] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:35,548] INFO : Plotted number of elements +rank0 [2026-11-15 04:04:36,854] INFO : Plotted jets +rank0 [2026-11-15 04:04:38,237] INFO : Plotted jet ratio +rank0 [2026-11-15 04:05:04,117] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:05:04,117] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:06:03,944] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:06:03,944] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:06:25,648] INFO : Step 12000/500000 rank0 | Train Loss: 12.6323 | Train Loss Unweighted: 1.8929 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.7227 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0702 | Regression_eta: 0.0088 | Regression_pt: 0.0689 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5966 | Classification_binary: 1.7227 | Regression_cos_phi: 1.7871 | Regression_energy: 1.4207 | Regression_eta: 2.8234 | Regression_pt: 1.3846 | Regression_sin_phi: 1.8972 | LR: 2.75e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0410 | max_loss: 0.0027 | max_backward: 0.0407 | max_optimizer_step: 0.0613 | max_tensorboard_logging: 0.1274 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3057.9 | uss_MB: 1831.4 | child_rss_MB: 22808.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:06:25,650] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:06:25,715] INFO : Running validation on rank0 +rank0 [2026-11-15 04:06:25,715] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:06:25,715] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:06:25,862] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:06:25,980] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:06:26,094] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:06:26,162] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:06:26,228] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:06:26,292] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:06:26,357] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:06:26,422] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:06:26,527] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:06:26,641] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:06:26,757] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:06:26,823] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:06:26,889] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:06:26,956] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:06:27,024] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:06:27,092] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:06:27,200] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:06:27,207] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:06:27,208] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:06:27,210] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:06:27,212] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:06:27,216] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:06:27,218] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:06:27,218] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:06:27,220] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:06:27,222] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:06:27,224] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:06:27,228] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:06:27,230] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:06:27,232] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:06:27,232] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:06:27,236] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:06:28,685] INFO : VALIDATION | Step=12000/500000 | Train Loss=12.6323 | Valid Loss=14.1165 | Train Loss Unweighted=1.8929 | Valid Loss Unweighted=2.2412 | Stale=7 | ETA=958.0m +rank0 [2026-11-15 04:06:28,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:06:28,762] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:06:28,767] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:28,818] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:06:29,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:06:29,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:06:29,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:06:30,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:06:30,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:06:30,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:06:30,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:06:31,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:06:31,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:06:31,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:06:31,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:06:32,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:06:32,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:06:32,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:06:32,814] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:06:33,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:06:33,095] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:06:33,103] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:33,674] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:06:34,056] INFO : Plotted number of elements +rank0 [2026-11-15 04:06:35,355] INFO : Plotted jets +rank0 [2026-11-15 04:06:36,735] INFO : Plotted jet ratio +rank0 [2026-11-15 04:07:15,955] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:07:15,955] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:16,373] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:08:16,373] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:24,685] INFO : Step 13000/500000 rank0 | Train Loss: 10.4660 | Train Loss Unweighted: 1.8990 | Train Loss Components: Classification: 0.0114 | Classification_binary: 1.7438 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0647 | Regression_eta: 0.0065 | Regression_pt: 0.0659 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4160 | Classification_binary: 1.7438 | Regression_cos_phi: 1.4441 | Regression_energy: 1.3083 | Regression_eta: 2.0837 | Regression_pt: 1.3254 | Regression_sin_phi: 1.1447 | LR: 2.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4671s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0607 | max_tensorboard_logging: 0.1220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3090.5 | uss_MB: 1864.0 | child_rss_MB: 22805.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:08:24,686] INFO : Running validation on rank0 +rank0 [2026-11-15 04:08:24,687] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:08:24,687] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:08:24,832] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:08:24,948] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:08:25,060] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:08:25,130] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:08:25,195] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:08:25,261] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:08:25,327] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:08:25,392] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:08:25,498] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:08:25,613] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:08:25,727] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:08:25,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:08:25,860] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:08:25,924] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:08:25,988] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:08:26,054] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:08:26,163] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:08:26,170] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:08:26,172] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:08:26,174] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:08:26,176] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:08:26,178] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:08:26,180] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:08:26,182] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:08:26,184] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:08:26,186] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:08:26,188] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:08:26,190] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:08:26,192] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:08:26,194] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:08:26,196] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:08:26,198] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:08:27,635] INFO : VALIDATION | Step=13000/500000 | Train Loss=10.4660 | Valid Loss=14.1912 | Train Loss Unweighted=1.8990 | Valid Loss Unweighted=2.2452 | Stale=8 | ETA=956.8m +rank0 [2026-11-15 04:08:27,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:08:27,709] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:08:27,715] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:27,765] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:08:28,252] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:08:28,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:08:28,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:08:29,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:08:29,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:08:29,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:08:29,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:08:30,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:08:30,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:08:30,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:08:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:08:31,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:08:31,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:08:31,532] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:08:31,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:08:32,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:08:32,062] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:08:32,067] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:32,637] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:33,017] INFO : Plotted number of elements +rank0 [2026-11-15 04:08:34,331] INFO : Plotted jets +rank0 [2026-11-15 04:08:35,736] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:28,240] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:09:28,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:23,589] INFO : Step 14000/500000 rank0 | Train Loss: 8.5803 | Train Loss Unweighted: 1.8046 | Train Loss Components: Classification: 0.0119 | Classification_binary: 1.6694 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0572 | Regression_eta: 0.0023 | Regression_pt: 0.0577 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.4747 | Classification_binary: 1.6694 | Regression_cos_phi: 1.4999 | Regression_energy: 1.1569 | Regression_eta: 0.7374 | Regression_pt: 1.1588 | Regression_sin_phi: 0.8832 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4784s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1220 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3114.0 | uss_MB: 1887.5 | child_rss_MB: 22811.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 15.6 +rank0 [2026-11-15 04:10:23,590] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:10:23,658] INFO : Running validation on rank0 +rank0 [2026-11-15 04:10:23,658] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:10:23,658] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:10:23,804] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:10:23,923] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:10:24,038] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:10:24,107] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:10:24,172] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:10:24,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:10:24,306] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:10:24,375] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:10:24,482] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:10:24,597] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:10:24,713] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:10:24,781] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:10:24,847] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:10:24,912] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:10:24,980] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:10:25,046] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:10:25,156] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:10:25,163] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:10:25,164] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:10:25,166] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:10:25,168] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:10:25,170] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:10:25,173] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:10:25,176] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:10:25,176] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:10:25,178] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:10:25,182] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:10:25,182] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:10:25,184] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:10:25,186] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:10:25,190] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:10:25,191] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:10:26,647] INFO : VALIDATION | Step=14000/500000 | Train Loss=8.5803 | Valid Loss=14.2052 | Train Loss Unweighted=1.8046 | Valid Loss Unweighted=2.2524 | Stale=9 | ETA=955.5m +rank0 [2026-11-15 04:10:26,647] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:10:26,720] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:10:26,724] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:26,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:10:27,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:10:27,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:10:27,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:10:28,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:10:28,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:10:28,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:10:28,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:10:29,033] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:10:29,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:10:29,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:10:29,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:10:30,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:10:30,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:10:30,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:10:30,780] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:10:31,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:10:31,068] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:10:31,073] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:31,646] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:32,026] INFO : Plotted number of elements +rank0 [2026-11-15 04:10:33,762] INFO : Plotted jets +rank0 [2026-11-15 04:10:34,784] INFO : Plotted jet ratio +rank0 [2026-11-15 04:10:40,518] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:10:40,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:40,344] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:11:40,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:22,014] INFO : Step 15000/500000 rank0 | Train Loss: 10.8163 | Train Loss Unweighted: 2.1410 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.9850 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0652 | Regression_eta: 0.0053 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6046 | Classification_binary: 1.9850 | Regression_cos_phi: 1.8253 | Regression_energy: 1.3200 | Regression_eta: 1.7023 | Regression_pt: 1.3090 | Regression_sin_phi: 1.0701 | LR: 3.17e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4692s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3139.8 | uss_MB: 1913.3 | child_rss_MB: 22808.1 | allocated_MB: 170.9 | reserved_MB: 674.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:12:22,016] INFO : Running validation on rank0 +rank0 [2026-11-15 04:12:22,016] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:12:22,016] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:12:22,161] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:12:22,277] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:12:22,392] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:12:22,461] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:12:22,527] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:12:22,594] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:12:22,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:12:22,729] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:12:22,835] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:12:22,948] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:12:23,063] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:12:23,130] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:12:23,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:12:23,258] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:12:23,324] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:12:23,389] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:12:23,496] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:12:23,503] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:12:23,505] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:12:23,507] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:12:23,508] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:12:23,510] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:12:23,514] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:12:23,514] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:12:23,516] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:12:23,519] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:12:23,521] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:12:23,523] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:12:23,524] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:12:23,526] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:12:23,529] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:12:23,531] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:12:24,983] INFO : VALIDATION | Step=15000/500000 | Train Loss=10.8163 | Valid Loss=14.2968 | Train Loss Unweighted=2.1410 | Valid Loss Unweighted=2.2472 | Stale=10 | ETA=953.7m +rank0 [2026-11-15 04:12:24,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:12:25,070] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:12:25,073] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:25,120] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:12:25,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:12:25,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:12:26,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:12:26,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:12:26,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:12:26,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:12:27,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:12:27,373] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:12:27,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:12:27,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:12:28,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:12:28,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:12:28,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:12:28,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:12:29,125] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:12:29,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:12:29,418] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:12:29,426] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:29,992] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:12:30,376] INFO : Plotted number of elements +rank0 [2026-11-15 04:12:31,689] INFO : Plotted jets +rank0 [2026-11-15 04:12:33,103] INFO : Plotted jet ratio +rank0 [2026-11-15 04:12:51,965] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:12:51,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:51,852] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:13:51,852] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:14:20,423] INFO : Step 16000/500000 rank0 | Train Loss: 10.7147 | Train Loss Unweighted: 2.0555 | Train Loss Components: Classification: 0.0103 | Classification_binary: 1.8865 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0734 | Regression_eta: 0.0054 | Regression_pt: 0.0725 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.2765 | Classification_binary: 1.8865 | Regression_cos_phi: 1.6796 | Regression_energy: 1.4858 | Regression_eta: 1.7471 | Regression_pt: 1.4577 | Regression_sin_phi: 1.1816 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4794s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3163.1 | uss_MB: 1936.7 | child_rss_MB: 22811.7 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:14:20,425] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:14:20,491] INFO : Running validation on rank0 +rank0 [2026-11-15 04:14:20,491] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:14:20,491] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:14:20,636] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:14:20,752] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:14:20,865] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:14:20,932] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:14:20,998] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:14:21,065] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:14:21,133] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:14:21,199] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:14:21,306] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:14:21,421] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:14:21,536] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:14:21,605] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:14:21,670] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:14:21,733] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:14:21,799] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:14:21,864] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:14:21,972] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:14:21,979] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:14:21,981] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:14:21,982] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:14:21,986] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:14:21,986] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:14:21,988] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:14:21,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:14:21,992] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:14:21,996] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:14:21,997] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:14:21,999] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:14:22,000] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:14:22,004] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:14:22,006] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:14:22,006] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:14:23,470] INFO : VALIDATION | Step=16000/500000 | Train Loss=10.7147 | Valid Loss=14.4796 | Train Loss Unweighted=2.0555 | Valid Loss Unweighted=2.2542 | Stale=11 | ETA=952.0m +rank0 [2026-11-15 04:14:23,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:14:23,546] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:14:23,553] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:23,603] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:24,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:24,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:24,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:24,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:25,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:25,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:25,600] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:25,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:26,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:26,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:26,611] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:26,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:27,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:27,355] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:27,601] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:27,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:27,904] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:14:27,909] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,483] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,864] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:30,593] INFO : Plotted jets +rank0 [2026-11-15 04:14:31,579] INFO : Plotted jet ratio +rank0 [2026-11-15 04:15:03,668] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:15:03,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:04,265] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:16:04,265] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:19,574] INFO : Step 17000/500000 rank0 | Train Loss: 11.4339 | Train Loss Unweighted: 2.0598 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.8778 | Regression_cos_phi: 0.0046 | Regression_energy: 0.0776 | Regression_eta: 0.0032 | Regression_pt: 0.0783 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6430 | Classification_binary: 1.8778 | Regression_cos_phi: 1.7495 | Regression_energy: 1.5713 | Regression_eta: 1.0262 | Regression_pt: 1.5726 | Regression_sin_phi: 1.9935 | LR: 3.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.4734s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0602 | max_tensorboard_logging: 0.1314 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3101.2 | uss_MB: 1874.8 | child_rss_MB: 22811.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:16:19,575] INFO : Running validation on rank0 +rank0 [2026-11-15 04:16:19,575] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:16:19,575] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:16:19,720] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:16:19,840] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:16:19,952] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:16:20,022] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:16:20,089] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:16:20,153] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:16:20,218] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:16:20,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:16:20,390] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:16:20,506] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:16:20,620] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:16:20,687] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:16:20,756] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:16:20,820] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:16:20,884] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:16:20,949] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:16:21,059] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:16:21,066] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:16:21,067] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:16:21,070] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:16:21,072] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:16:21,074] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:16:21,076] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:16:21,078] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:16:21,080] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:16:21,082] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:16:21,083] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:16:21,086] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:16:21,087] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:16:21,089] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:16:21,092] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:16:21,095] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:16:22,530] INFO : VALIDATION | Step=17000/500000 | Train Loss=11.4339 | Valid Loss=14.5255 | Train Loss Unweighted=2.0598 | Valid Loss Unweighted=2.2621 | Stale=12 | ETA=950.5m +rank0 [2026-11-15 04:16:22,530] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:16:22,605] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:16:22,620] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:22,670] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:16:23,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:16:23,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:16:23,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:16:23,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:16:24,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:16:24,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:16:24,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:16:24,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:16:25,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:16:25,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:16:25,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:16:25,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:16:26,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:16:26,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:16:26,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:16:26,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:16:27,000] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:16:27,008] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:27,580] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:16:27,960] INFO : Plotted number of elements +rank0 [2026-11-15 04:16:29,280] INFO : Plotted jets +rank0 [2026-11-15 04:16:30,687] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:16,035] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:17:16,035] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:16,128] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:18:16,129] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:18,241] INFO : Step 18000/500000 rank0 | Train Loss: 11.3289 | Train Loss Unweighted: 2.1826 | Train Loss Components: Classification: 0.0130 | Classification_binary: 2.0009 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0788 | Regression_eta: 0.0055 | Regression_pt: 0.0771 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6136 | Classification_binary: 2.0009 | Regression_cos_phi: 1.5928 | Regression_energy: 1.5951 | Regression_eta: 1.7632 | Regression_pt: 1.5500 | Regression_sin_phi: 1.2131 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4815s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0617 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3206.4 | uss_MB: 1979.9 | child_rss_MB: 22810.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:18:18,243] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:18:18,310] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:18,310] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:18,310] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:18,455] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:18:18,574] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:18,688] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:18,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:18,824] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:18,889] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:18,956] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:19,024] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:19,132] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:19,248] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:19,363] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:19,429] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:19,494] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:19,558] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:19,623] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:19,688] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:19,798] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:19,806] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:19,808] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:19,810] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:19,812] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:19,814] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:19,815] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:19,818] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:19,820] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:19,822] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:19,824] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:19,825] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:19,828] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:19,830] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:19,832] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:19,834] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:21,299] INFO : VALIDATION | Step=18000/500000 | Train Loss=11.3289 | Valid Loss=14.4844 | Train Loss Unweighted=2.1826 | Valid Loss Unweighted=2.2684 | Stale=13 | ETA=948.9m +rank0 [2026-11-15 04:18:21,299] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:18:21,375] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:18:21,379] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:21,430] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:21,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:22,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:22,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:22,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:22,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:23,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:23,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:23,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:23,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:24,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:24,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:24,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:24,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:25,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:25,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:25,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:25,717] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:18:25,722] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:26,295] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:26,673] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:27,980] INFO : Plotted jets +rank0 [2026-11-15 04:18:29,404] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:27,897] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:19:27,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:16,598] INFO : Step 19000/500000 rank0 | Train Loss: 13.5408 | Train Loss Unweighted: 2.1380 | Train Loss Components: Classification: 0.0175 | Classification_binary: 1.8788 | Regression_cos_phi: 0.0028 | Regression_energy: 0.1129 | Regression_eta: 0.0068 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 2.1726 | Classification_binary: 1.8788 | Regression_cos_phi: 1.0526 | Regression_energy: 2.2848 | Regression_eta: 2.2060 | Regression_pt: 2.3115 | Regression_sin_phi: 1.6345 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1255 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3235.4 | uss_MB: 2009.0 | child_rss_MB: 22807.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 04:20:16,599] INFO : Running validation on rank0 +rank0 [2026-11-15 04:20:16,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:20:16,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:20:16,744] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:20:16,861] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:20:16,974] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:20:17,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:20:17,111] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:20:17,175] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:20:17,241] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:20:17,307] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:20:17,412] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:20:17,529] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:20:17,643] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:20:17,711] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:20:17,778] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:20:17,845] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:20:17,912] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:20:17,979] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:20:18,092] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:20:18,098] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:20:18,100] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:20:18,102] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:20:18,104] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:20:18,108] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:20:18,110] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:20:18,112] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:20:18,114] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:20:18,114] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:20:18,116] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:20:18,120] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:20:18,122] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:20:18,122] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:20:18,124] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:20:18,127] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:20:19,587] INFO : VALIDATION | Step=19000/500000 | Train Loss=13.5408 | Valid Loss=14.6540 | Train Loss Unweighted=2.1380 | Valid Loss Unweighted=2.2740 | Stale=14 | ETA=947.0m +rank0 [2026-11-15 04:20:19,587] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:20:19,662] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:20:19,665] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:19,716] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:20:20,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:20:20,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:20:20,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:20:21,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:20:21,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:20:21,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:20:21,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:20:22,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:20:22,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:20:22,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:20:22,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:20:23,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:20:23,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:20:23,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:20:23,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:20:24,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:20:24,067] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:20:24,076] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:24,651] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:25,033] INFO : Plotted number of elements +rank0 [2026-11-15 04:20:26,342] INFO : Plotted jets +rank0 [2026-11-15 04:20:27,758] INFO : Plotted jet ratio +rank0 [2026-11-15 04:20:39,670] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:20:39,670] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:39,967] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:21:39,968] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:15,756] INFO : Step 20000/500000 rank0 | Train Loss: 11.8463 | Train Loss Unweighted: 2.2414 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.0211 | Regression_cos_phi: 0.0036 | Regression_energy: 0.0984 | Regression_eta: 0.0045 | Regression_pt: 0.0977 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5114 | Classification_binary: 2.0211 | Regression_cos_phi: 1.3833 | Regression_energy: 1.9910 | Regression_eta: 1.4344 | Regression_pt: 1.9637 | Regression_sin_phi: 1.5413 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4832s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0025 | max_backward: 0.0389 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3175.0 | uss_MB: 1948.6 | child_rss_MB: 22810.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 15.6 +rank0 [2026-11-15 04:22:15,758] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:22:15,824] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:15,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:15,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:22:15,978] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:22:16,102] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:22:16,219] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:22:16,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:22:16,357] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:22:16,424] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:22:16,491] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:22:16,557] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:22:16,662] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:22:16,775] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:22:16,886] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:22:16,953] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:22:17,019] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:22:17,085] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:22:17,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:22:17,215] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:22:17,324] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:22:17,330] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:22:17,333] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:22:17,335] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:22:17,337] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:22:17,339] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:22:17,340] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:22:17,343] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:22:17,345] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:22:17,347] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:22:17,349] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:22:17,351] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:22:17,353] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:22:17,355] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:22:17,357] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:22:17,360] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:22:18,813] INFO : VALIDATION | Step=20000/500000 | Train Loss=11.8463 | Valid Loss=14.8133 | Train Loss Unweighted=2.2414 | Valid Loss Unweighted=2.2824 | Stale=15 | ETA=945.4m +rank0 [2026-11-15 04:22:18,813] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:22:18,892] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:22:18,896] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:18,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:22:19,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:22:19,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:22:19,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:22:20,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:22:20,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:22:20,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:22:20,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:22:21,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:22:21,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:22:21,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:22:21,927] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:22:22,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:22:22,422] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:22:22,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:22:22,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:22:23,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:22:23,228] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:22:23,234] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:23,795] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:22:24,173] INFO : Plotted number of elements +rank0 [2026-11-15 04:22:25,479] INFO : Plotted jets +rank0 [2026-11-15 04:22:26,892] INFO : Plotted jet ratio +rank0 [2026-11-15 04:22:51,814] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:22:51,814] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:51,934] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:23:51,934] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:14,450] INFO : Step 21000/500000 rank0 | Train Loss: 11.0032 | Train Loss Unweighted: 2.2319 | Train Loss Components: Classification: 0.0078 | Classification_binary: 2.0639 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0700 | Regression_eta: 0.0055 | Regression_pt: 0.0765 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 0.9611 | Classification_binary: 2.0639 | Regression_cos_phi: 1.6690 | Regression_energy: 1.4162 | Regression_eta: 1.7701 | Regression_pt: 1.5365 | Regression_sin_phi: 1.5866 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3288.1 | uss_MB: 2061.7 | child_rss_MB: 22812.6 | allocated_MB: 170.4 | reserved_MB: 676.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:24:14,451] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:14,451] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:14,451] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:14,597] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:24:14,716] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:14,830] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:14,900] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:14,966] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:15,033] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:15,101] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:15,167] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:15,275] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:15,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:15,506] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:15,573] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:15,639] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:15,703] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:15,769] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:15,834] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:15,944] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:15,951] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:15,953] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:15,954] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:15,957] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:15,958] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:15,962] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:15,963] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:15,965] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:15,968] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:15,968] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:15,970] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:15,972] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:15,976] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:15,978] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:15,978] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:17,443] INFO : VALIDATION | Step=21000/500000 | Train Loss=11.0032 | Valid Loss=14.8529 | Train Loss Unweighted=2.2319 | Valid Loss Unweighted=2.2808 | Stale=16 | ETA=943.6m +rank0 [2026-11-15 04:24:17,444] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:17,520] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:17,524] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:17,575] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:18,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:18,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:18,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:18,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:19,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:19,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:19,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:19,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:20,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:20,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:20,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:20,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:21,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:21,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:21,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:21,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:21,865] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:24:21,873] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:22,440] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:22,819] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:24,149] INFO : Plotted jets +rank0 [2026-11-15 04:24:25,593] INFO : Plotted jet ratio +rank0 [2026-11-15 04:25:03,894] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:25:03,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:04,149] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:26:04,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:13,370] INFO : Step 22000/500000 rank0 | Train Loss: 10.2378 | Train Loss Unweighted: 2.2843 | Train Loss Components: Classification: 0.0111 | Classification_binary: 2.0957 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0842 | Regression_eta: 0.0025 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3747 | Classification_binary: 2.0957 | Regression_cos_phi: 1.0333 | Regression_energy: 1.7029 | Regression_eta: 0.8172 | Regression_pt: 1.6936 | Regression_sin_phi: 1.5204 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4913s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3312.7 | uss_MB: 2086.2 | child_rss_MB: 22809.2 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:26:13,372] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:26:13,439] INFO : Running validation on rank0 +rank0 [2026-11-15 04:26:13,439] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:26:13,439] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:26:13,584] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:26:13,699] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:26:13,811] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:26:13,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:26:13,944] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:26:14,008] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:26:14,076] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:26:14,141] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:26:14,246] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:26:14,358] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:26:14,470] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:26:14,537] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:26:14,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:26:14,668] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:26:14,735] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:26:14,801] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:26:14,914] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:26:14,921] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:26:14,921] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:26:14,924] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:26:14,926] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:26:14,929] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:26:14,931] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:26:14,933] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:26:14,935] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:26:14,937] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:26:14,939] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:26:14,940] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:26:14,943] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:26:14,945] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:26:14,947] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:26:14,949] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:26:16,396] INFO : VALIDATION | Step=22000/500000 | Train Loss=10.2378 | Valid Loss=14.9806 | Train Loss Unweighted=2.2843 | Valid Loss Unweighted=2.2841 | Stale=17 | ETA=941.9m +rank0 [2026-11-15 04:26:16,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:26:16,471] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:26:16,476] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:16,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:26:17,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:26:17,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:26:17,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:26:17,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:26:18,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:26:18,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:26:18,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:26:18,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:26:19,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:26:19,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:26:19,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:26:19,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:26:20,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:26:20,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:26:20,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:26:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:26:20,824] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:26:20,835] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:21,415] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:21,799] INFO : Plotted number of elements +rank0 [2026-11-15 04:26:23,554] INFO : Plotted jets +rank0 [2026-11-15 04:26:24,552] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:16,339] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:27:16,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:12,146] INFO : Step 23000/500000 rank0 | Train Loss: 8.0623 | Train Loss Unweighted: 1.9944 | Train Loss Components: Classification: 0.0111 | Classification_binary: 1.8564 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0589 | Regression_eta: 0.0013 | Regression_pt: 0.0616 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3803 | Classification_binary: 1.8564 | Regression_cos_phi: 0.6644 | Regression_energy: 1.1923 | Regression_eta: 0.4261 | Regression_pt: 1.2384 | Regression_sin_phi: 1.3046 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4814s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0024 | max_backward: 0.0385 | max_optimizer_step: 0.0609 | max_tensorboard_logging: 0.1291 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3323.2 | uss_MB: 2096.7 | child_rss_MB: 22811.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:28:12,148] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:12,148] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:12,148] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:12,294] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:28:12,413] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:12,526] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:12,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:12,662] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:12,728] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:12,794] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:12,861] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:12,968] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:13,087] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:13,203] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:13,269] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:13,335] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:13,399] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:13,465] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:13,530] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:13,642] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:13,649] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:13,650] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:13,653] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:13,655] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:13,657] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:13,659] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:13,661] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:13,663] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:13,666] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:13,666] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:13,669] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:13,670] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:13,672] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:13,674] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:13,677] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:15,129] INFO : VALIDATION | Step=23000/500000 | Train Loss=8.0623 | Valid Loss=15.1393 | Train Loss Unweighted=1.9944 | Valid Loss Unweighted=2.2871 | Stale=18 | ETA=940.1m +rank0 [2026-11-15 04:28:15,130] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:15,206] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:15,209] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:15,260] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:15,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:16,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:16,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:16,527] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:16,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:17,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:17,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:17,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:17,777] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:18,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:18,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:18,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:18,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:19,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:19,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:19,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:19,565] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:28:19,576] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:20,144] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:20,524] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:21,838] INFO : Plotted jets +rank0 [2026-11-15 04:28:23,266] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:28,110] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:28:28,110] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:27,981] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:29:27,982] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:10,491] INFO : Step 24000/500000 rank0 | Train Loss: 10.3893 | Train Loss Unweighted: 1.9571 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.8108 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0589 | Regression_eta: 0.0060 | Regression_pt: 0.0619 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5902 | Classification_binary: 1.8108 | Regression_cos_phi: 1.2409 | Regression_energy: 1.1920 | Regression_eta: 1.9329 | Regression_pt: 1.2440 | Regression_sin_phi: 1.3785 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5087s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1454 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3358.9 | uss_MB: 2132.5 | child_rss_MB: 22809.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:10,493] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:30:10,559] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:10,559] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:10,560] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:10,704] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:30:10,822] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:10,935] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:11,003] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:11,070] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:11,136] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:11,201] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:11,266] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:11,371] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:11,483] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:11,596] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:11,664] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:11,730] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:11,796] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:11,863] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:11,930] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:12,042] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:12,050] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:12,052] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:12,054] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:12,056] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:12,058] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:12,061] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:12,062] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:12,064] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:12,067] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:12,069] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:12,071] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:12,073] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:12,075] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:12,077] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:12,080] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:13,539] INFO : VALIDATION | Step=24000/500000 | Train Loss=10.3893 | Valid Loss=15.0848 | Train Loss Unweighted=1.9571 | Valid Loss Unweighted=2.3022 | Stale=19 | ETA=938.2m +rank0 [2026-11-15 04:30:13,539] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:13,614] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:13,617] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:13,669] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:14,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:14,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:14,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:14,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:15,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:15,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:15,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:15,931] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:16,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:16,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:16,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:16,945] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:17,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:17,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:17,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:17,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:17,971] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:30:17,977] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:18,549] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:18,936] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:20,225] INFO : Plotted jets +rank0 [2026-11-15 04:30:21,644] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:39,688] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:30:39,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:39,245] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:31:39,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:08,418] INFO : Step 25000/500000 rank0 | Train Loss: 10.9924 | Train Loss Unweighted: 2.0393 | Train Loss Components: Classification: 0.0157 | Classification_binary: 1.8905 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0616 | Regression_eta: 0.0058 | Regression_pt: 0.0583 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.9481 | Classification_binary: 1.8905 | Regression_cos_phi: 1.6976 | Regression_energy: 1.2467 | Regression_eta: 1.8607 | Regression_pt: 1.1720 | Regression_sin_phi: 1.1767 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3385.9 | uss_MB: 2159.5 | child_rss_MB: 22813.5 | allocated_MB: 170.4 | reserved_MB: 676.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:32:08,420] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:08,420] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:08,420] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:08,563] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:32:08,684] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:08,796] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:08,864] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:08,929] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:08,993] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:09,060] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:09,126] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:09,232] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:09,348] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:09,463] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:09,531] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:09,598] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:09,664] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:09,731] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:09,797] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:09,910] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:09,916] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:09,919] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:09,922] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:09,922] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:09,924] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:09,926] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:09,928] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:09,930] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:09,932] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:09,934] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:09,936] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:09,939] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:09,940] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:09,944] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:09,946] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:11,400] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.9924 | Valid Loss=15.2046 | Train Loss Unweighted=2.0393 | Valid Loss Unweighted=2.3036 | Stale=20 | ETA=936.1m +rank0 [2026-11-15 04:32:11,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:11,476] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:11,481] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:11,532] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:12,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:12,286] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:12,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:12,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:13,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:13,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:13,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:13,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:14,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:14,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:14,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:14,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:15,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:15,315] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:15,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:15,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:15,845] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:32:15,851] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:16,430] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:16,813] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:18,139] INFO : Plotted jets +rank0 [2026-11-15 04:32:19,559] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:51,046] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:32:51,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:50,751] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:33:50,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:06,986] INFO : Step 26000/500000 rank0 | Train Loss: 11.4067 | Train Loss Unweighted: 2.0718 | Train Loss Components: Classification: 0.0131 | Classification_binary: 1.9035 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0707 | Regression_eta: 0.0020 | Regression_pt: 0.0714 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6285 | Classification_binary: 1.9035 | Regression_cos_phi: 1.5703 | Regression_energy: 1.4306 | Regression_eta: 0.6343 | Regression_pt: 1.4346 | Regression_sin_phi: 2.8050 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0636 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3406.3 | uss_MB: 2179.9 | child_rss_MB: 22811.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:06,988] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:34:07,053] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:07,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:07,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:07,200] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:34:07,317] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:07,430] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:07,498] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:07,563] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:07,629] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:07,696] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:07,763] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:07,869] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:07,985] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:08,103] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:08,170] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:08,235] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:08,299] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:08,365] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:08,430] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:08,539] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:08,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:08,547] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:08,551] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:08,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:08,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:08,557] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:08,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:08,561] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:08,563] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:08,565] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:08,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:08,569] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:08,571] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:08,573] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:08,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:10,020] INFO : VALIDATION | Step=26000/500000 | Train Loss=11.4067 | Valid Loss=15.3003 | Train Loss Unweighted=2.0718 | Valid Loss Unweighted=2.3100 | Stale=21 | ETA=934.3m +rank0 [2026-11-15 04:34:10,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:10,092] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:10,095] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:10,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:10,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:10,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:11,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:11,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:11,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:11,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:12,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:12,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:12,661] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:12,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:13,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:13,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:13,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:13,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:14,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:14,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:14,465] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:34:14,471] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:15,048] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:15,429] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:16,734] INFO : Plotted jets +rank0 [2026-11-15 04:34:18,163] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:02,577] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:35:02,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:02,728] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:36:02,728] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:05,688] INFO : Step 27000/500000 rank0 | Train Loss: 10.5402 | Train Loss Unweighted: 1.9638 | Train Loss Components: Classification: 0.0121 | Classification_binary: 1.8033 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0690 | Regression_eta: 0.0071 | Regression_pt: 0.0666 | Regression_sin_phi: 0.0018 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5016 | Classification_binary: 1.8033 | Regression_cos_phi: 1.5321 | Regression_energy: 1.3961 | Regression_eta: 2.2731 | Regression_pt: 1.3385 | Regression_sin_phi: 0.6956 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0026 | max_backward: 0.0402 | max_optimizer_step: 0.0630 | max_tensorboard_logging: 0.1256 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3432.1 | uss_MB: 2205.6 | child_rss_MB: 22812.4 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:36:05,690] INFO : Running validation on rank0 +rank0 [2026-11-15 04:36:05,690] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:36:05,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:36:05,834] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:36:05,951] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:36:06,065] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:36:06,134] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:36:06,200] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:36:06,264] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:36:06,329] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:36:06,394] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:36:06,499] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:36:06,612] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:36:06,727] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:36:06,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:36:06,862] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:36:06,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:36:06,994] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:36:07,063] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:36:07,176] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:36:07,184] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:36:07,184] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:36:07,187] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:36:07,190] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:36:07,191] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:36:07,194] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:36:07,196] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:36:07,198] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:36:07,200] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:36:07,202] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:36:07,204] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:36:07,206] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:36:07,208] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:36:07,210] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:36:07,212] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:36:08,648] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.5402 | Valid Loss=15.4868 | Train Loss Unweighted=1.9638 | Valid Loss Unweighted=2.3059 | Stale=22 | ETA=932.4m +rank0 [2026-11-15 04:36:08,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:36:08,723] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:36:08,726] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:08,778] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:36:09,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:36:09,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:36:09,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:36:10,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:36:10,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:36:10,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:36:10,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:36:11,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:36:11,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:36:11,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:36:11,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:36:12,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:36:12,272] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:36:12,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:36:12,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:36:12,982] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:36:13,040] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:36:13,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:13,618] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:14,002] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:15,290] INFO : Plotted jets +rank0 [2026-11-15 04:36:16,708] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:14,018] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:37:14,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:03,465] INFO : Step 28000/500000 rank0 | Train Loss: 9.7251 | Train Loss Unweighted: 2.0203 | Train Loss Components: Classification: 0.0131 | Classification_binary: 1.8789 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0589 | Regression_eta: 0.0028 | Regression_pt: 0.0591 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6242 | Classification_binary: 1.8789 | Regression_cos_phi: 1.4336 | Regression_energy: 1.1916 | Regression_eta: 0.9122 | Regression_pt: 1.1868 | Regression_sin_phi: 1.4978 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1259 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3455.0 | uss_MB: 2228.6 | child_rss_MB: 22812.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 13.6 +rank0 [2026-11-15 04:38:03,467] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:38:03,533] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:03,533] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:03,533] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:03,685] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:38:03,806] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:03,923] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:03,994] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:04,062] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:04,129] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:04,197] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:04,264] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:04,369] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:04,484] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:04,596] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:04,664] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:04,729] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:04,793] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:04,858] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:04,922] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:05,033] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:05,039] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:05,042] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:05,043] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:05,047] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:05,047] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:05,050] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:05,053] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:05,055] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:05,057] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:05,059] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:05,060] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:05,063] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:05,065] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:05,067] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:05,069] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:06,511] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.7251 | Valid Loss=15.5625 | Train Loss Unweighted=2.0203 | Valid Loss Unweighted=2.3225 | Stale=23 | ETA=930.3m +rank0 [2026-11-15 04:38:06,512] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:06,593] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:06,596] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:06,644] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:07,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:07,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:07,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:07,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:08,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:08,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:08,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:08,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:09,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:09,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:09,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:09,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:10,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:10,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:10,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:10,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:10,954] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:38:10,960] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:11,527] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:11,910] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:13,232] INFO : Plotted jets +rank0 [2026-11-15 04:38:14,663] INFO : Plotted jet ratio +rank0 [2026-11-15 04:38:25,640] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:38:25,640] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:25,311] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:39:25,311] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:01,572] INFO : Step 29000/500000 rank0 | Train Loss: 10.0288 | Train Loss Unweighted: 2.3887 | Train Loss Components: Classification: 0.0112 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0700 | Regression_eta: 0.0042 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3873 | Classification_binary: 2.2258 | Regression_cos_phi: 0.9708 | Regression_energy: 1.4173 | Regression_eta: 1.3666 | Regression_pt: 1.4433 | Regression_sin_phi: 1.2178 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3482.5 | uss_MB: 2256.0 | child_rss_MB: 22813.3 | allocated_MB: 170.4 | reserved_MB: 700.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:01,574] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:01,574] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:01,574] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:01,719] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:40:01,837] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:01,949] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:02,018] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:02,085] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:02,152] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:02,218] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:02,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:02,392] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:02,507] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:02,622] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:02,690] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:02,758] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:02,825] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:02,892] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:02,959] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:03,074] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:03,081] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:03,084] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:03,084] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:03,088] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:03,090] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:03,092] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:03,094] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:03,096] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:03,098] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:03,100] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:03,102] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:03,104] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:03,105] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:03,107] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:03,109] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:04,551] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.0288 | Valid Loss=15.7530 | Train Loss Unweighted=2.3887 | Valid Loss Unweighted=2.3303 | Stale=24 | ETA=928.3m +rank0 [2026-11-15 04:40:04,551] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:04,628] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:04,631] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:04,683] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:05,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:05,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:05,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:05,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:06,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:06,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:06,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:06,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:07,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:07,428] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:07,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:07,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:08,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:08,447] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:08,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:08,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:08,978] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:40:08,984] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,562] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,948] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:11,253] INFO : Plotted jets +rank0 [2026-11-15 04:40:12,670] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:36,798] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:40:36,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:36,508] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:41:36,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:59,672] INFO : Step 30000/500000 rank0 | Train Loss: 8.4215 | Train Loss Unweighted: 1.9282 | Train Loss Components: Classification: 0.0135 | Classification_binary: 1.7875 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0608 | Regression_eta: 0.0022 | Regression_pt: 0.0595 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6765 | Classification_binary: 1.7875 | Regression_cos_phi: 0.8244 | Regression_energy: 1.2305 | Regression_eta: 0.7078 | Regression_pt: 1.1963 | Regression_sin_phi: 0.9985 | LR: 6.58e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3506.2 | uss_MB: 2279.8 | child_rss_MB: 22814.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:59,673] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:41:59,740] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:59,740] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:59,740] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:59,891] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:42:00,009] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:00,128] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:00,196] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:00,263] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:00,330] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:00,398] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:00,464] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:00,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:00,682] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:00,795] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:00,861] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:00,925] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:00,989] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:01,055] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:01,121] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:01,234] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:01,241] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:01,243] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:01,244] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:01,248] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:01,248] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:01,252] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:01,254] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:01,256] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:01,256] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:01,260] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:01,262] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:01,264] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:01,266] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:01,266] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:01,269] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:02,720] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.4215 | Valid Loss=15.7263 | Train Loss Unweighted=1.9282 | Valid Loss Unweighted=2.3297 | Stale=25 | ETA=926.3m +rank0 [2026-11-15 04:42:02,721] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:02,797] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:02,802] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:02,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:03,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:03,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:03,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:04,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:04,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:04,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:04,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:05,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:05,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:05,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:05,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:06,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:06,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:06,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:06,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:07,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:07,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:42:07,144] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:07,709] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:08,091] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:09,406] INFO : Plotted jets +rank0 [2026-11-15 04:42:10,833] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:48,190] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:42:48,190] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:47,962] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:43:47,962] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:57,912] INFO : Step 31000/500000 rank0 | Train Loss: 9.0567 | Train Loss Unweighted: 2.3107 | Train Loss Components: Classification: 0.0113 | Classification_binary: 2.1404 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0746 | Regression_eta: 0.0027 | Regression_pt: 0.0776 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3955 | Classification_binary: 2.1404 | Regression_cos_phi: 0.8884 | Regression_energy: 1.5099 | Regression_eta: 0.8848 | Regression_pt: 1.5591 | Regression_sin_phi: 0.6786 | LR: 6.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4851s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0388 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1268 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3446.3 | uss_MB: 2219.8 | child_rss_MB: 22809.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:43:57,913] INFO : Running validation on rank0 +rank0 [2026-11-15 04:43:57,913] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:43:57,913] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:43:58,062] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:43:58,182] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:43:58,295] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:43:58,364] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:43:58,429] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:43:58,493] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:43:58,558] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:43:58,622] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:43:58,726] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:43:58,840] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:43:58,952] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:43:59,020] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:43:59,089] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:43:59,156] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:43:59,223] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:43:59,289] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:43:59,401] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:43:59,408] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:43:59,409] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:43:59,412] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:43:59,414] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:43:59,416] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:43:59,417] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:43:59,419] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:43:59,421] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:43:59,423] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:43:59,426] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:43:59,427] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:43:59,429] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:43:59,431] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:43:59,433] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:43:59,437] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:00,883] INFO : VALIDATION | Step=31000/500000 | Train Loss=9.0567 | Valid Loss=15.7236 | Train Loss Unweighted=2.3107 | Valid Loss Unweighted=2.3357 | Stale=26 | ETA=924.3m +rank0 [2026-11-15 04:44:00,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:00,954] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:00,958] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:01,010] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:01,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:01,763] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:02,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:02,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:02,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:02,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:03,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:03,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:03,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:03,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:04,021] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:04,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:04,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:04,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:05,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:05,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:05,324] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:44:05,330] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:05,912] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:06,304] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:07,594] INFO : Plotted jets +rank0 [2026-11-15 04:44:09,007] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:59,524] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:44:59,524] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:56,126] INFO : Step 32000/500000 rank0 | Train Loss: 10.7088 | Train Loss Unweighted: 2.0974 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9360 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0685 | Regression_eta: 0.0034 | Regression_pt: 0.0680 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.6150 | Classification_binary: 1.9360 | Regression_cos_phi: 1.8698 | Regression_energy: 1.3862 | Regression_eta: 1.0838 | Regression_pt: 1.3663 | Regression_sin_phi: 1.4517 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4796s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0621 | max_tensorboard_logging: 0.1235 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3555.4 | uss_MB: 2329.0 | child_rss_MB: 22806.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:45:56,128] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:45:56,194] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:56,194] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:56,194] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:56,336] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:45:56,454] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:56,567] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:56,636] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:56,702] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:56,767] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:56,834] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:56,900] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:57,006] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:57,125] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:57,240] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:57,309] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:57,374] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:57,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:57,503] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:57,568] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:57,677] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:57,684] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:57,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:57,687] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:57,690] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:57,691] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:57,695] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:57,695] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:57,697] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:57,701] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:57,702] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:57,703] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:57,706] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:57,708] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:57,709] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:57,713] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:59,159] INFO : VALIDATION | Step=32000/500000 | Train Loss=10.7088 | Valid Loss=15.9452 | Train Loss Unweighted=2.0974 | Valid Loss Unweighted=2.3487 | Stale=27 | ETA=922.3m +rank0 [2026-11-15 04:45:59,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:59,233] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:59,236] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:59,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:59,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:00,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:00,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:00,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:00,840] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:01,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:01,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:01,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:01,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:02,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:02,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:02,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:02,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:03,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:03,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:03,567] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:03,655] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:46:03,661] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:04,235] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:04,613] INFO : Plotted number of elements +rank0 [2026-11-15 04:46:05,914] INFO : Plotted jets +rank0 [2026-11-15 04:46:07,312] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:11,268] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:46:11,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:11,222] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:47:11,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:47:54,753] INFO : Step 33000/500000 rank0 | Train Loss: 6.8837 | Train Loss Unweighted: 1.9010 | Train Loss Components: Classification: 0.0075 | Classification_binary: 1.7671 | Regression_cos_phi: 0.0015 | Regression_energy: 0.0599 | Regression_eta: 0.0016 | Regression_pt: 0.0617 | Regression_sin_phi: 0.0017 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 0.9281 | Classification_binary: 1.7671 | Regression_cos_phi: 0.5688 | Regression_energy: 1.2130 | Regression_eta: 0.5010 | Regression_pt: 1.2401 | Regression_sin_phi: 0.6656 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5037s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0649 | max_tensorboard_logging: 0.1399 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3588.3 | uss_MB: 2426.1 | child_rss_MB: 22807.1 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:47:54,754] INFO : Running validation on rank0 +rank0 [2026-11-15 04:47:54,754] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:47:54,754] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:47:54,900] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:47:55,018] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:47:55,135] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:47:55,203] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:47:55,269] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:47:55,334] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:47:55,401] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:47:55,468] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:47:55,576] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:47:55,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:47:55,808] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:47:55,876] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:47:55,943] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:47:56,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:47:56,081] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:47:56,149] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:47:56,262] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:47:56,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:47:56,270] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:47:56,280] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:47:56,280] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:47:56,282] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:47:56,285] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:47:56,286] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:47:56,290] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:47:56,291] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:47:56,294] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:47:56,295] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:47:56,298] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:47:56,300] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:47:56,301] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:47:56,304] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:57,772] INFO : VALIDATION | Step=33000/500000 | Train Loss=6.8837 | Valid Loss=15.9740 | Train Loss Unweighted=1.9010 | Valid Loss Unweighted=2.3558 | Stale=28 | ETA=920.4m +rank0 [2026-11-15 04:47:57,772] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:57,859] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:47:57,862] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:57,912] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:58,420] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:58,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:58,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:59,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:59,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:59,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:59,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:00,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:00,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:00,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:00,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:01,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:01,484] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:01,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:01,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:02,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:02,270] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:48:02,276] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:02,851] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:03,240] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:04,553] INFO : Plotted jets +rank0 [2026-11-15 04:48:05,977] INFO : Plotted jet ratio +rank0 [2026-11-15 04:48:23,295] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-15 04:48:23,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:23,216] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-15 04:49:23,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:53,502] INFO : Step 34000/500000 rank0 | Train Loss: 9.4840 | Train Loss Unweighted: 2.1458 | Train Loss Components: Classification: 0.0107 | Classification_binary: 1.9642 | Regression_cos_phi: 0.0023 | Regression_energy: 0.0824 | Regression_eta: 0.0037 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.3231 | Classification_binary: 1.9642 | Regression_cos_phi: 0.8855 | Regression_energy: 1.6677 | Regression_eta: 1.1855 | Regression_pt: 1.6159 | Regression_sin_phi: 0.8421 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0396 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1532 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3596.6 | uss_MB: 2434.4 | child_rss_MB: 22803.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:49:53,504] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-15 04:49:53,570] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:53,570] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:53,570] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:53,713] INFO : saved batch0_epoch34000.parquet +rank0 [2026-11-15 04:49:53,832] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:53,944] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:54,013] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:54,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:54,144] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:54,210] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:54,276] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:54,381] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:54,497] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:54,610] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:54,678] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:54,745] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:54,811] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:54,878] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:54,945] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:55,056] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:55,063] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:55,066] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:55,068] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:55,070] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:55,072] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:55,074] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:55,075] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:55,078] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:55,081] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:55,083] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:55,083] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:55,085] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:55,089] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:55,090] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:55,092] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:56,535] INFO : VALIDATION | Step=34000/500000 | Train Loss=9.4840 | Valid Loss=16.2660 | Train Loss Unweighted=2.1458 | Valid Loss Unweighted=2.3731 | Stale=29 | ETA=918.6m +rank0 [2026-11-15 04:49:56,536] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:56,621] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:56,629] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:56,677] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:57,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:57,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:57,705] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:57,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:58,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:58,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:58,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:58,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:59,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:59,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:59,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:59,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:50:00,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:50:00,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:50:00,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:50:00,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:50:00,987] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:50:00,993] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:50:01,578] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_34000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:50:01,958] INFO : Plotted number of elements +rank0 [2026-11-15 04:50:03,265] INFO : Plotted jets +rank0 [2026-11-15 04:50:04,685] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:35,216] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-15 04:50:35,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:35,184] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-15 04:51:35,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:52,278] INFO : Step 35000/500000 rank0 | Train Loss: 10.8430 | Train Loss Unweighted: 2.0802 | Train Loss Components: Classification: 0.0122 | Classification_binary: 1.8954 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0795 | Regression_eta: 0.0069 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 123.9109 | Classification_binary: 1.0000 | Regression_cos_phi: 382.3351 | Regression_energy: 20.2364 | Regression_eta: 322.2987 | Regression_pt: 20.0966 | Regression_sin_phi: 396.8552 | Task Weighted Losses: Classification: 1.5136 | Classification_binary: 1.8954 | Regression_cos_phi: 0.8530 | Regression_energy: 1.6094 | Regression_eta: 2.2143 | Regression_pt: 1.6295 | Regression_sin_phi: 1.1278 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5405s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1735 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3640.2 | uss_MB: 2477.9 | child_rss_MB: 22812.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 19.6 +rank0 [2026-11-15 04:51:52,280] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:52,280] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:52,280] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:52,423] INFO : saved batch0_epoch35000.parquet +rank0 [2026-11-15 04:51:52,539] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:52,653] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:52,721] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:52,787] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:52,852] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:52,919] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:52,987] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:53,095] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:53,211] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:53,327] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:53,396] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:53,461] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:53,525] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:53,590] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:53,655] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:53,764] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:53,770] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:53,772] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:53,776] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:53,778] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:53,778] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:53,781] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:53,782] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:53,784] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:53,788] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:53,788] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:53,792] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:53,794] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:53,796] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:53,798] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:53,800] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:55,253] INFO : VALIDATION | Step=35000/500000 | Train Loss=10.8430 | Valid Loss=16.2588 | Train Loss Unweighted=2.0802 | Valid Loss Unweighted=2.3791 | Stale=30 | ETA=916.7m +rank0 [2026-11-15 04:51:55,254] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:55,350] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:55,358] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:55,406] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:55,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:56,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:56,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:56,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:56,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:57,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:57,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:57,690] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:57,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:58,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:58,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:58,721] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:58,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:59,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:59,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:59,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:59,749] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-15 04:51:59,755] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:00,335] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_15mm_seed12345_20261115_034231_666046/preds_step_35000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:00,722] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:02,041] INFO : Plotted jets +rank0 [2026-11-15 04:52:03,487] INFO : Plotted jet ratio +rank0 [2026-11-15 04:52:03,837] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:52:03,837] INFO : Training completed. Total time on device 0: 69.143min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..2c1cba7103dc3b2f43169c24e60cb0da24c07d85 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17d212c8e44300afa3ef98727866d1c2d6b5179977ffb5ce69df905962672185 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-32000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-32000.pth new file mode 100644 index 0000000000000000000000000000000000000000..88e8ea70718d17a20e32aae3179f3d8bfdda87c8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-32000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e82c5d5aa4ec7d0354321709c1e589dec7f324b07dbf48a292afef9c47e205c +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..0e5919f7e03b6f969adef7e844f9346fa6ee1334 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 2.1457810401916504, "Classification": 0.014380941167473793, "Regression_pt": 0.0827244222164154, "Regression_eta": 0.004315992817282677, "Regression_sin_phi": 0.004792259074747562, "Regression_cos_phi": 0.004977609496563673, "Regression_energy": 0.08200698345899582, "Total": 12.481261253356934, "Total_unweighted": 2.3389792484231293}, "train_group": {}, "valid": {"Classification_binary": 1.9416080713272095, "Classification": 0.015721933916211128, "Regression_pt": 0.09323990345001221, "Regression_eta": 0.0058494084514677525, "Regression_sin_phi": 0.00526057742536068, "Regression_cos_phi": 0.00536452978849411, "Regression_energy": 0.09258780628442764, "Total": 13.734907150268555, "Total_unweighted": 2.159632230643183, "metrics/particle/class_1/efficiency": 0.9523009061813354, "metrics/particle/class_1/pid_accuracy": 0.9860607981681824, "metrics/particle/class_2/efficiency": 0.5489192008972168, "metrics/particle/class_2/pid_accuracy": 0.8383419513702393, "metrics/particle/class_3/efficiency": 0.8424968719482422, "metrics/particle/class_3/pid_accuracy": 0.9894483089447021, "metrics/particle/class_4/efficiency": 0.8675078749656677, "metrics/particle/class_4/pid_accuracy": 0.578181803226471, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7536231875419617, "metrics/particle/count/bias_mean": 0.2840000092983246, "metrics/particle/count/mae": 2.859999895095825, "metrics/particle/count/prediction_mean": 96.01200103759766, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02600095048546791, "metrics/particle/event/energy_response_mean": 1.0021557807922363, "metrics/particle/event/met_abs_error": 4.324276447296143, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026460010558366776, "metrics/particle/event/scalar_pt_response_mean": 1.0057300329208374, "metrics/particle/event/vector_pt_closure": 0.02851472608745098, "metrics/particle/matched/delta_eta_abs_mean": 0.008723504841327667, "metrics/particle/matched/delta_phi_abs_mean": 0.00962176825851202, "metrics/particle/matched/delta_r_mean": 0.014536323957145214, "metrics/particle/matched/energy_relative_abs_mean": 0.08610010147094727, "metrics/particle/matched/pid_accuracy": 0.9730013608932495, "metrics/particle/matched/pt_relative_abs_mean": 0.08110088855028152, "metrics/particle/matching/duplicate_fraction": 0.026496687904000282, "metrics/particle/matching/efficiency": 0.8744359016418457, "metrics/particle/matching/f1": 0.8731406927108765, "metrics/particle/matching/purity": 0.8718493580818176, "metrics/particle/matching/target_coverage_dr0p05": 0.853501558303833, "metrics/particle/matching/target_coverage_dr0p10": 0.9032257795333862, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7688032984733582, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.901972234249115}, "step_train_time": 0.8349735736846924, "step_valid_time": 1.8400066220056033, "step_total_time": 78.44073343276978, "elapsed_training_loop_seconds": 78.44073343276978, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002626754535831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06075106077026071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.96, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9574036511156186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.952124072825354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9572881355932203, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546991210277215, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548213081591369, "step_test_time": 3.19010096299462, "step_plot_time": 2.9410071120073553, "step_validation_cycle_time": 8.423510904001887} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..222e593dc2508e282791a93588132b7706a059f6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 1.859182596206665, "Classification": 0.011191366240382195, "Regression_pt": 0.09413892030715942, "Regression_eta": 0.003152035176753998, "Regression_sin_phi": 0.0035406758543103933, "Regression_cos_phi": 0.003371387254446745, "Regression_energy": 0.09680531919002533, "Total": 10.836201667785645, "Total_unweighted": 2.071382300229743}, "train_group": {}, "valid": {"Classification_binary": 1.9706428050994873, "Classification": 0.01586761511862278, "Regression_pt": 0.09503030776977539, "Regression_eta": 0.005880852695554495, "Regression_sin_phi": 0.005194809753447771, "Regression_cos_phi": 0.005542796570807695, "Regression_energy": 0.09447019547224045, "Total": 13.909921646118164, "Total_unweighted": 2.192629382479936, "metrics/particle/class_1/efficiency": 0.9545132517814636, "metrics/particle/class_1/pid_accuracy": 0.9865566492080688, "metrics/particle/class_2/efficiency": 0.5432309508323669, "metrics/particle/class_2/pid_accuracy": 0.8314136266708374, "metrics/particle/class_3/efficiency": 0.8400811553001404, "metrics/particle/class_3/pid_accuracy": 0.9896480441093445, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.595588207244873, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.7644230723381042, "metrics/particle/count/bias_mean": 0.37599998712539673, "metrics/particle/count/mae": 2.888000011444092, "metrics/particle/count/prediction_mean": 96.10399627685547, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0267960075289011, "metrics/particle/event/energy_response_mean": 1.0025749206542969, "metrics/particle/event/met_abs_error": 4.276885509490967, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026009002700448036, "metrics/particle/event/scalar_pt_response_mean": 1.0040287971496582, "metrics/particle/event/vector_pt_closure": 0.02749066799879074, "metrics/particle/matched/delta_eta_abs_mean": 0.00858489703387022, "metrics/particle/matched/delta_phi_abs_mean": 0.009775545448064804, "metrics/particle/matched/delta_r_mean": 0.014525641687214375, "metrics/particle/matched/energy_relative_abs_mean": 0.0866381973028183, "metrics/particle/matched/pid_accuracy": 0.9734640121459961, "metrics/particle/matched/pt_relative_abs_mean": 0.0810236781835556, "metrics/particle/matching/duplicate_fraction": 0.025305919349193573, "metrics/particle/matching/efficiency": 0.8739345073699951, "metrics/particle/matching/f1": 0.8722215294837952, "metrics/particle/matching/purity": 0.8705152869224548, "metrics/particle/matching/target_coverage_dr0p05": 0.8533762097358704, "metrics/particle/matching/target_coverage_dr0p10": 0.9032257795333862, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7690539956092834, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9014708399772644}, "step_train_time": 0.836845874786377, "step_valid_time": 2.2000076879921835, "step_total_time": 842.3180403709412, "elapsed_training_loop_seconds": 842.3180403709412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001643496470707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06191697202816748, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547908232118758, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954468802698145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04520917678812416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534412955465587, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9531197301854974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9541469993256911, "step_test_time": 3.326692942006048, "step_plot_time": 2.8727678600116633, "step_validation_cycle_time": 8.819345413998235} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..bd11b744ca489143bdf3753b30ebc501f5cda4f0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 1.8938757181167603, "Classification": 0.010122996754944324, "Regression_pt": 0.06971718370914459, "Regression_eta": 0.002727353945374489, "Regression_sin_phi": 0.002380192279815674, "Regression_cos_phi": 0.0032396395690739155, "Regression_energy": 0.06682782620191574, "Total": 9.009811401367188, "Total_unweighted": 2.048890910577029}, "train_group": {}, "valid": {"Classification_binary": 1.9721498489379883, "Classification": 0.01593627780675888, "Regression_pt": 0.09523043036460876, "Regression_eta": 0.005928507540374994, "Regression_sin_phi": 0.00530365901067853, "Regression_cos_phi": 0.00558726629242301, "Regression_energy": 0.09479473531246185, "Total": 14.00438404083252, "Total_unweighted": 2.1949307252652943, "metrics/particle/class_1/efficiency": 0.9554867148399353, "metrics/particle/class_1/pid_accuracy": 0.9861072301864624, "metrics/particle/class_2/efficiency": 0.5364050269126892, "metrics/particle/class_2/pid_accuracy": 0.8335101008415222, "metrics/particle/class_3/efficiency": 0.8403710722923279, "metrics/particle/class_3/pid_accuracy": 0.9895366430282593, "metrics/particle/class_4/efficiency": 0.8611987233161926, "metrics/particle/class_4/pid_accuracy": 0.6117216348648071, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7536231875419617, "metrics/particle/count/bias_mean": 0.5080000162124634, "metrics/particle/count/mae": 2.9719998836517334, "metrics/particle/count/prediction_mean": 96.23600006103516, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.027288515120744705, "metrics/particle/event/energy_response_mean": 1.0063626766204834, "metrics/particle/event/met_abs_error": 4.6053571701049805, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02706335298717022, "metrics/particle/event/scalar_pt_response_mean": 1.0090997219085693, "metrics/particle/event/vector_pt_closure": 0.02948843687772751, "metrics/particle/matched/delta_eta_abs_mean": 0.008340275846421719, "metrics/particle/matched/delta_phi_abs_mean": 0.009960087947547436, "metrics/particle/matched/delta_r_mean": 0.01450402569025755, "metrics/particle/matched/energy_relative_abs_mean": 0.08585451543331146, "metrics/particle/matched/pid_accuracy": 0.9734665751457214, "metrics/particle/matched/pt_relative_abs_mean": 0.08067089319229126, "metrics/particle/matching/duplicate_fraction": 0.02747412584722042, "metrics/particle/matching/efficiency": 0.8740180730819702, "metrics/particle/matching/f1": 0.8717051148414612, "metrics/particle/matching/purity": 0.8694043755531311, "metrics/particle/matching/target_coverage_dr0p05": 0.8552147746086121, "metrics/particle/matching/target_coverage_dr0p10": 0.9046465158462524, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7697643041610718, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9009276032447815}, "step_train_time": 0.8670105934143066, "step_valid_time": 2.0645667589997174, "step_total_time": 928.8070914745331, "elapsed_training_loop_seconds": 928.8070914745331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042745545724123, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06437335090454932, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9548821548821549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9555256064690026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04511784511784512, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9535353535353536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.954177897574124, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.507236875986564, "step_plot_time": 3.2168072009953903, "step_validation_cycle_time": 9.287822333004442} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..e8efa835391406aaa520ad62ddf65d3645148635 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 1.5423260927200317, "Classification": 0.008823170326650143, "Regression_pt": 0.06077602505683899, "Regression_eta": 0.002157819690182805, "Regression_sin_phi": 0.0018619031179696321, "Regression_cos_phi": 0.0025956418830901384, "Regression_energy": 0.06057589501142502, "Total": 7.5517683029174805, "Total_unweighted": 1.6791165478061885}, "train_group": {}, "valid": {"Classification_binary": 1.9799432754516602, "Classification": 0.01619330607354641, "Regression_pt": 0.09603836387395859, "Regression_eta": 0.0058325547724962234, "Regression_sin_phi": 0.005142867099493742, "Regression_cos_phi": 0.005505167879164219, "Regression_energy": 0.09558712691068649, "Total": 13.954695701599121, "Total_unweighted": 2.204242662061006, "metrics/particle/class_1/efficiency": 0.9560176730155945, "metrics/particle/class_1/pid_accuracy": 0.9841710925102234, "metrics/particle/class_2/efficiency": 0.5426621437072754, "metrics/particle/class_2/pid_accuracy": 0.8249475955963135, "metrics/particle/class_3/efficiency": 0.8422069549560547, "metrics/particle/class_3/pid_accuracy": 0.990018367767334, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.6433823704719543, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7777777910232544, "metrics/particle/count/bias_mean": 0.6639999747276306, "metrics/particle/count/mae": 2.8320000171661377, "metrics/particle/count/prediction_mean": 96.39199829101562, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.026931172236800194, "metrics/particle/event/energy_response_mean": 1.0064266920089722, "metrics/particle/event/met_abs_error": 4.420836448669434, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025858474895358086, "metrics/particle/event/scalar_pt_response_mean": 1.0076202154159546, "metrics/particle/event/vector_pt_closure": 0.028633322566747665, "metrics/particle/matched/delta_eta_abs_mean": 0.008491599932312965, "metrics/particle/matched/delta_phi_abs_mean": 0.009888137690722942, "metrics/particle/matched/delta_r_mean": 0.014557416550815105, "metrics/particle/matched/energy_relative_abs_mean": 0.08681292831897736, "metrics/particle/matched/pid_accuracy": 0.9728904366493225, "metrics/particle/matched/pt_relative_abs_mean": 0.08099592477083206, "metrics/particle/matching/duplicate_fraction": 0.026101751253008842, "metrics/particle/matching/efficiency": 0.8754805326461792, "metrics/particle/matching/f1": 0.8724547028541565, "metrics/particle/matching/purity": 0.8694497346878052, "metrics/particle/matching/target_coverage_dr0p05": 0.8564683198928833, "metrics/particle/matching/target_coverage_dr0p10": 0.9054403901100159, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7703075408935547, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9028497338294983}, "step_train_time": 0.8399045467376709, "step_valid_time": 2.012581864997628, "step_total_time": 1015.3071525096893, "elapsed_training_loop_seconds": 1015.3071525096893, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0041462871134579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06127974365130351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568151147098516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564924114671164, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043184885290148446, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9554655870445344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551433389544688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.2784401470125886, "step_plot_time": 2.8719686650001677, "step_validation_cycle_time": 8.60384943599638} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..9688935f0841476ce38c524dc801e1c19ae843c2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 2.122955799102783, "Classification": 0.011245914734899998, "Regression_pt": 0.06959699839353561, "Regression_eta": 0.003133545396849513, "Regression_sin_phi": 0.0061656334437429905, "Regression_cos_phi": 0.004743450786918402, "Regression_energy": 0.07038988173007965, "Total": 11.549736976623535, "Total_unweighted": 2.2882312235888094}, "train_group": {}, "valid": {"Classification_binary": 1.9861879348754883, "Classification": 0.01604267582297325, "Regression_pt": 0.09684114158153534, "Regression_eta": 0.005876410286873579, "Regression_sin_phi": 0.0053166612051427364, "Regression_cos_phi": 0.0055951341055333614, "Regression_energy": 0.09637252241373062, "Total": 14.08582878112793, "Total_unweighted": 2.212232480291277, "metrics/particle/class_1/efficiency": 0.9550442695617676, "metrics/particle/class_1/pid_accuracy": 0.9857301712036133, "metrics/particle/class_2/efficiency": 0.5420932769775391, "metrics/particle/class_2/pid_accuracy": 0.8373557329177856, "metrics/particle/class_3/efficiency": 0.8424002528190613, "metrics/particle/class_3/pid_accuracy": 0.9893324375152588, "metrics/particle/class_4/efficiency": 0.8548895716667175, "metrics/particle/class_4/pid_accuracy": 0.61623615026474, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7922705411911011, "metrics/particle/count/bias_mean": 0.7239999771118164, "metrics/particle/count/mae": 2.9560000896453857, "metrics/particle/count/prediction_mean": 96.4520034790039, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02725079283118248, "metrics/particle/event/energy_response_mean": 1.003731608390808, "metrics/particle/event/met_abs_error": 4.296095848083496, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027093814685940742, "metrics/particle/event/scalar_pt_response_mean": 1.0051039457321167, "metrics/particle/event/vector_pt_closure": 0.028086164966225624, "metrics/particle/matched/delta_eta_abs_mean": 0.00886301789432764, "metrics/particle/matched/delta_phi_abs_mean": 0.009748842567205429, "metrics/particle/matched/delta_r_mean": 0.01472470723092556, "metrics/particle/matched/energy_relative_abs_mean": 0.08663862198591232, "metrics/particle/matched/pid_accuracy": 0.9737834930419922, "metrics/particle/matched/pt_relative_abs_mean": 0.08130564540624619, "metrics/particle/matching/duplicate_fraction": 0.02683199942111969, "metrics/particle/matching/efficiency": 0.8750209212303162, "metrics/particle/matching/f1": 0.8717244267463684, "metrics/particle/matching/purity": 0.8684527277946472, "metrics/particle/matching/target_coverage_dr0p05": 0.8556326031684875, "metrics/particle/matching/target_coverage_dr0p10": 0.906777560710907, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7691793441772461, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9024319052696228}, "step_train_time": 0.8846774101257324, "step_valid_time": 2.0410772080067545, "step_total_time": 1100.654269695282, "elapsed_training_loop_seconds": 1100.654269695282, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00331561726385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06262800222044285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9561107359891965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9554655870445344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04388926401080351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.952124072825354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534098582039163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9527665317139001, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548213081591369, "step_test_time": 3.2127777010027785, "step_plot_time": 2.8680032970005414, "step_validation_cycle_time": 8.536689462998766} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..f333809cb3f95318f697c8ba32e6a07bb44f9e7b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 1.9457625150680542, "Classification": 0.011400151066482067, "Regression_pt": 0.07384657114744186, "Regression_eta": 0.004456895869225264, "Regression_sin_phi": 0.0028854073025286198, "Regression_cos_phi": 0.0028690388426184654, "Regression_energy": 0.0730576440691948, "Total": 10.097710609436035, "Total_unweighted": 2.1142782233655453}, "train_group": {}, "valid": {"Classification_binary": 1.9851516485214233, "Classification": 0.01612459123134613, "Regression_pt": 0.09642145037651062, "Regression_eta": 0.005895324517041445, "Regression_sin_phi": 0.0052809747867286205, "Regression_cos_phi": 0.00563867948949337, "Regression_energy": 0.09603768587112427, "Total": 14.090088844299316, "Total_unweighted": 2.210550354793668, "metrics/particle/class_1/efficiency": 0.9539822936058044, "metrics/particle/class_1/pid_accuracy": 0.9867346882820129, "metrics/particle/class_2/efficiency": 0.5398179888725281, "metrics/particle/class_2/pid_accuracy": 0.8314014673233032, "metrics/particle/class_3/efficiency": 0.8409507870674133, "metrics/particle/class_3/pid_accuracy": 0.9894289374351501, "metrics/particle/class_4/efficiency": 0.8485804200172424, "metrics/particle/class_4/pid_accuracy": 0.613382875919342, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7584540843963623, "metrics/particle/count/bias_mean": 0.3400000035762787, "metrics/particle/count/mae": 2.931999921798706, "metrics/particle/count/prediction_mean": 96.06800079345703, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02772960253059864, "metrics/particle/event/energy_response_mean": 1.0039104223251343, "metrics/particle/event/met_abs_error": 4.384928226470947, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02732267789542675, "metrics/particle/event/scalar_pt_response_mean": 1.0048627853393555, "metrics/particle/event/vector_pt_closure": 0.028506096452474594, "metrics/particle/matched/delta_eta_abs_mean": 0.008300334215164185, "metrics/particle/matched/delta_phi_abs_mean": 0.009965579956769943, "metrics/particle/matched/delta_r_mean": 0.014473644085228443, "metrics/particle/matched/energy_relative_abs_mean": 0.08691607415676117, "metrics/particle/matched/pid_accuracy": 0.9737421274185181, "metrics/particle/matched/pt_relative_abs_mean": 0.0810098946094513, "metrics/particle/matching/duplicate_fraction": 0.02610650844871998, "metrics/particle/matching/efficiency": 0.8736419677734375, "metrics/particle/matching/f1": 0.8720932602882385, "metrics/particle/matching/purity": 0.8705500364303589, "metrics/particle/matching/target_coverage_dr0p05": 0.854504406452179, "metrics/particle/matching/target_coverage_dr0p10": 0.9041450619697571, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7681764960289001, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9012201428413391}, "step_train_time": 0.929213285446167, "step_valid_time": 1.9970241929986514, "step_total_time": 1186.7137529850006, "elapsed_training_loop_seconds": 1186.7137529850006, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030496053037399, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06381234093175359, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574611748818366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568151147098516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042538825118163405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9554956169925826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567859554355166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.956140350877193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.208749229001114, "step_plot_time": 2.8433299860043917, "step_validation_cycle_time": 8.427398850995814} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..cbb532b665ca26a2528d056203b0c39bb032da49 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 2.225499153137207, "Classification": 0.016535423696041107, "Regression_pt": 0.09042859077453613, "Regression_eta": 0.004881896544247866, "Regression_sin_phi": 0.003361629555001855, "Regression_cos_phi": 0.0029860257636755705, "Regression_energy": 0.08778152614831924, "Total": 12.050737380981445, "Total_unweighted": 2.431474245619029}, "train_group": {}, "valid": {"Classification_binary": 2.0005526542663574, "Classification": 0.016065608710050583, "Regression_pt": 0.0968669205904007, "Regression_eta": 0.005971008911728859, "Regression_sin_phi": 0.0053994301706552505, "Regression_cos_phi": 0.005686928518116474, "Regression_energy": 0.0966123640537262, "Total": 14.206456184387207, "Total_unweighted": 2.2271549152210355, "metrics/particle/class_1/efficiency": 0.9543362855911255, "metrics/particle/class_1/pid_accuracy": 0.9858123064041138, "metrics/particle/class_2/efficiency": 0.5375426411628723, "metrics/particle/class_2/pid_accuracy": 0.8391534686088562, "metrics/particle/class_3/efficiency": 0.8424002528190613, "metrics/particle/class_3/pid_accuracy": 0.9907088875770569, "metrics/particle/class_4/efficiency": 0.8611987233161926, "metrics/particle/class_4/pid_accuracy": 0.6190476417541504, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7632850408554077, "metrics/particle/count/bias_mean": 0.41999998688697815, "metrics/particle/count/mae": 2.8519999980926514, "metrics/particle/count/prediction_mean": 96.14800262451172, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.028530074283480644, "metrics/particle/event/energy_response_mean": 1.0021088123321533, "metrics/particle/event/met_abs_error": 4.336552619934082, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028008675202727318, "metrics/particle/event/scalar_pt_response_mean": 1.0034321546554565, "metrics/particle/event/vector_pt_closure": 0.02819625847041607, "metrics/particle/matched/delta_eta_abs_mean": 0.008444743230938911, "metrics/particle/matched/delta_phi_abs_mean": 0.009942163713276386, "metrics/particle/matched/delta_r_mean": 0.014547894708812237, "metrics/particle/matched/energy_relative_abs_mean": 0.08593930304050446, "metrics/particle/matched/pid_accuracy": 0.9742438197135925, "metrics/particle/matched/pt_relative_abs_mean": 0.08039337396621704, "metrics/particle/matching/duplicate_fraction": 0.026584016159176826, "metrics/particle/matching/efficiency": 0.8744359016418457, "metrics/particle/matching/f1": 0.8725218176841736, "metrics/particle/matching/purity": 0.8706161379814148, "metrics/particle/matching/target_coverage_dr0p05": 0.856593668460846, "metrics/particle/matching/target_coverage_dr0p10": 0.9044376015663147, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7704328894615173, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9023901224136353}, "step_train_time": 0.8630502223968506, "step_valid_time": 2.2637368849973427, "step_total_time": 1270.7304463386536, "elapsed_training_loop_seconds": 1270.7304463386536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001695871798268, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06532220984978732, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547297297297297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537630779615255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04527027027027027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9514497639919083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533783783783784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524130948363145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9527983816587997, "step_test_time": 3.2496521470020525, "step_plot_time": 2.905457243003184, "step_validation_cycle_time": 8.85398724800325} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..debcae8e116b5ddcf926c8e71c0a008d6b99d0ba --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 2.0373177528381348, "Classification": 0.015914546325802803, "Regression_pt": 0.07788301259279251, "Regression_eta": 0.008965115062892437, "Regression_sin_phi": 0.006512922700494528, "Regression_cos_phi": 0.003168318187817931, "Regression_energy": 0.07665563374757767, "Total": 13.97418212890625, "Total_unweighted": 2.2264173014555126}, "train_group": {}, "valid": {"Classification_binary": 1.9994550943374634, "Classification": 0.016117578372359276, "Regression_pt": 0.09672339260578156, "Regression_eta": 0.006071916781365871, "Regression_sin_phi": 0.0052196308970451355, "Regression_cos_phi": 0.005704106763005257, "Regression_energy": 0.09643150866031647, "Total": 14.181507110595703, "Total_unweighted": 2.225723228417337, "metrics/particle/class_1/efficiency": 0.9553982019424438, "metrics/particle/class_1/pid_accuracy": 0.9856428503990173, "metrics/particle/class_2/efficiency": 0.5375426411628723, "metrics/particle/class_2/pid_accuracy": 0.8359788656234741, "metrics/particle/class_3/efficiency": 0.8393081426620483, "metrics/particle/class_3/pid_accuracy": 0.9898687601089478, "metrics/particle/class_4/efficiency": 0.85173499584198, "metrics/particle/class_4/pid_accuracy": 0.6370370388031006, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7439613342285156, "metrics/particle/count/bias_mean": 0.41200000047683716, "metrics/particle/count/mae": 2.947999954223633, "metrics/particle/count/prediction_mean": 96.13999938964844, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.026969287544488907, "metrics/particle/event/energy_response_mean": 1.0015828609466553, "metrics/particle/event/met_abs_error": 4.380063056945801, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026836121454834938, "metrics/particle/event/scalar_pt_response_mean": 1.0031862258911133, "metrics/particle/event/vector_pt_closure": 0.028432348743081093, "metrics/particle/matched/delta_eta_abs_mean": 0.008624576963484287, "metrics/particle/matched/delta_phi_abs_mean": 0.010044319555163383, "metrics/particle/matched/delta_r_mean": 0.014774627983570099, "metrics/particle/matched/energy_relative_abs_mean": 0.08584056049585342, "metrics/particle/matched/pid_accuracy": 0.9737370610237122, "metrics/particle/matched/pt_relative_abs_mean": 0.08075643330812454, "metrics/particle/matching/duplicate_fraction": 0.0260869562625885, "metrics/particle/matching/efficiency": 0.8734748363494873, "metrics/particle/matching/f1": 0.8715991973876953, "metrics/particle/matching/purity": 0.8697316646575928, "metrics/particle/matching/target_coverage_dr0p05": 0.8533344268798828, "metrics/particle/matching/target_coverage_dr0p10": 0.9034765362739563, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7692629098892212, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9016379714012146}, "step_train_time": 0.7719943523406982, "step_valid_time": 2.0704130770027405, "step_total_time": 1356.3634161949158, "elapsed_training_loop_seconds": 1356.3634161949158, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011438484396133, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06174094822325649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9554956169925826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9567859554355166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956140350877193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04321404456448346, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534726904922455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9547602970965564, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9541160593792173, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9554956169925826, "step_test_time": 3.2662180379993515, "step_plot_time": 2.8343947940011276, "step_validation_cycle_time": 8.545237580998219} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..6572a08c972f6b759d7b57fa70c6bc5716dfaf8b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 2.128796339035034, "Classification": 0.013159831054508686, "Regression_pt": 0.08332405239343643, "Regression_eta": 0.0027585593052208424, "Regression_sin_phi": 0.002709823427721858, "Regression_cos_phi": 0.002684238599613309, "Regression_energy": 0.08369208872318268, "Total": 10.183920860290527, "Total_unweighted": 2.317124932538718}, "train_group": {}, "valid": {"Classification_binary": 2.0017738342285156, "Classification": 0.016206517815589905, "Regression_pt": 0.09812598675489426, "Regression_eta": 0.005997058004140854, "Regression_sin_phi": 0.00524133862927556, "Regression_cos_phi": 0.005674554966390133, "Regression_energy": 0.09759310632944107, "Total": 14.217653274536133, "Total_unweighted": 2.2306123967282474, "metrics/particle/class_1/efficiency": 0.9553097486495972, "metrics/particle/class_1/pid_accuracy": 0.9836035370826721, "metrics/particle/class_2/efficiency": 0.5472127199172974, "metrics/particle/class_2/pid_accuracy": 0.8295218348503113, "metrics/particle/class_3/efficiency": 0.837182343006134, "metrics/particle/class_3/pid_accuracy": 0.9898430109024048, "metrics/particle/class_4/efficiency": 0.8485804200172424, "metrics/particle/class_4/pid_accuracy": 0.654275119304657, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.7788461446762085, "metrics/particle/count/bias_mean": 0.3919999897480011, "metrics/particle/count/mae": 2.9200000762939453, "metrics/particle/count/prediction_mean": 96.12000274658203, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02736976556479931, "metrics/particle/event/energy_response_mean": 1.0057421922683716, "metrics/particle/event/met_abs_error": 4.47855806350708, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02704121731221676, "metrics/particle/event/scalar_pt_response_mean": 1.008304238319397, "metrics/particle/event/vector_pt_closure": 0.028820496052503586, "metrics/particle/matched/delta_eta_abs_mean": 0.008724872022867203, "metrics/particle/matched/delta_phi_abs_mean": 0.01030008029192686, "metrics/particle/matched/delta_r_mean": 0.015063702128827572, "metrics/particle/matched/energy_relative_abs_mean": 0.08619371801614761, "metrics/particle/matched/pid_accuracy": 0.9728203415870667, "metrics/particle/matched/pt_relative_abs_mean": 0.08096501976251602, "metrics/particle/matching/duplicate_fraction": 0.026633374392986298, "metrics/particle/matching/efficiency": 0.873224139213562, "metrics/particle/matching/f1": 0.8714398741722107, "metrics/particle/matching/purity": 0.8696629405021667, "metrics/particle/matching/target_coverage_dr0p05": 0.8534179925918579, "metrics/particle/matching/target_coverage_dr0p10": 0.9041032791137695, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7677168846130371, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.899882972240448}, "step_train_time": 0.8312115669250488, "step_valid_time": 2.0553237159911077, "step_total_time": 1440.2585203647614, "elapsed_training_loop_seconds": 1440.2585203647614, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033873807956863, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06404776522939915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1479, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1409, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1406, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9501011463250169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526707234617985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9513841998649561, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04732927653820149, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9480782198246797, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506423258958756, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.949358541525996, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9501011463250169, "step_test_time": 3.2058088469930226, "step_plot_time": 2.882051774999127, "step_validation_cycle_time": 8.584245666992501} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..8f20d828271d946ad3502b48abb3b638b1e74d76 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 1.8363803625106812, "Classification": 0.010942632332444191, "Regression_pt": 0.0672689825296402, "Regression_eta": 0.007001888006925583, "Regression_sin_phi": 0.0032480459194630384, "Regression_cos_phi": 0.003956233151257038, "Regression_energy": 0.06660512089729309, "Total": 11.101152420043945, "Total_unweighted": 1.9954032653477043}, "train_group": {}, "valid": {"Classification_binary": 2.0117239952087402, "Classification": 0.01626184955239296, "Regression_pt": 0.09766343981027603, "Regression_eta": 0.006031927652657032, "Regression_sin_phi": 0.0052668992429971695, "Regression_cos_phi": 0.005749484058469534, "Regression_energy": 0.09727484732866287, "Total": 14.268678665161133, "Total_unweighted": 2.239972442854196, "metrics/particle/class_1/efficiency": 0.9552212357521057, "metrics/particle/class_1/pid_accuracy": 0.9840652346611023, "metrics/particle/class_2/efficiency": 0.5386803150177002, "metrics/particle/class_2/pid_accuracy": 0.8352692723274231, "metrics/particle/class_3/efficiency": 0.8366025686264038, "metrics/particle/class_3/pid_accuracy": 0.9894894957542419, "metrics/particle/class_4/efficiency": 0.8422712683677673, "metrics/particle/class_4/pid_accuracy": 0.6554307341575623, "metrics/particle/class_5/efficiency": 0.9903846383094788, "metrics/particle/class_5/pid_accuracy": 0.7669903039932251, "metrics/particle/count/bias_mean": 0.492000013589859, "metrics/particle/count/mae": 2.940000057220459, "metrics/particle/count/prediction_mean": 96.22000122070312, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.029899071902036667, "metrics/particle/event/energy_response_mean": 1.008375644683838, "metrics/particle/event/met_abs_error": 4.664186000823975, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028982941061258316, "metrics/particle/event/scalar_pt_response_mean": 1.009264588356018, "metrics/particle/event/vector_pt_closure": 0.029538936913013458, "metrics/particle/matched/delta_eta_abs_mean": 0.008526086807250977, "metrics/particle/matched/delta_phi_abs_mean": 0.010083597153425217, "metrics/particle/matched/delta_r_mean": 0.014722183346748352, "metrics/particle/matched/energy_relative_abs_mean": 0.08644556254148483, "metrics/particle/matched/pid_accuracy": 0.9732177257537842, "metrics/particle/matched/pt_relative_abs_mean": 0.08108780533075333, "metrics/particle/matching/duplicate_fraction": 0.027229266241192818, "metrics/particle/matching/efficiency": 0.872137725353241, "metrics/particle/matching/f1": 0.8699022531509399, "metrics/particle/matching/purity": 0.8676782250404358, "metrics/particle/matching/target_coverage_dr0p05": 0.8524987697601318, "metrics/particle/matching/target_coverage_dr0p10": 0.9039779305458069, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7664633393287659, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9001337289810181}, "step_train_time": 0.8499643802642822, "step_valid_time": 2.3079862669983413, "step_total_time": 1525.8347797393799, "elapsed_training_loop_seconds": 1525.8347797393799, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004858891953079, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0654337698019185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9516129032258065, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9532144059239314, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04838709677419355, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495967741935484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511948838774823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548213081591369, "step_test_time": 3.560034351001377, "step_plot_time": 3.3460767020005733, "step_validation_cycle_time": 9.73291977700137} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..2752d905b09c16dd2e60d95d0fb75ec624829178 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 1.940293788909912, "Classification": 0.011893738061189651, "Regression_pt": 0.0692281574010849, "Regression_eta": 0.0019771468359977007, "Regression_sin_phi": 0.0028922059573233128, "Regression_cos_phi": 0.0017425082623958588, "Regression_energy": 0.0670621395111084, "Total": 8.652897834777832, "Total_unweighted": 2.095089684939012}, "train_group": {}, "valid": {"Classification_binary": 2.0104284286499023, "Classification": 0.01635197550058365, "Regression_pt": 0.09824594110250473, "Regression_eta": 0.006098869722336531, "Regression_sin_phi": 0.005290970206260681, "Regression_cos_phi": 0.005888956133276224, "Regression_energy": 0.09774000197649002, "Total": 14.384455680847168, "Total_unweighted": 2.240045143291354, "metrics/particle/class_1/efficiency": 0.9550442695617676, "metrics/particle/class_1/pid_accuracy": 0.9847108721733093, "metrics/particle/class_2/efficiency": 0.5403867959976196, "metrics/particle/class_2/pid_accuracy": 0.8326315879821777, "metrics/particle/class_3/efficiency": 0.8375688195228577, "metrics/particle/class_3/pid_accuracy": 0.9896169900894165, "metrics/particle/class_4/efficiency": 0.8485804200172424, "metrics/particle/class_4/pid_accuracy": 0.643122673034668, "metrics/particle/class_5/efficiency": 0.9903846383094788, "metrics/particle/class_5/pid_accuracy": 0.791262149810791, "metrics/particle/count/bias_mean": 0.4399999976158142, "metrics/particle/count/mae": 2.871999979019165, "metrics/particle/count/prediction_mean": 96.16799926757812, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02729654312133789, "metrics/particle/event/energy_response_mean": 1.0042908191680908, "metrics/particle/event/met_abs_error": 4.289574146270752, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0272661279886961, "metrics/particle/event/scalar_pt_response_mean": 1.0055245161056519, "metrics/particle/event/vector_pt_closure": 0.02808145247399807, "metrics/particle/matched/delta_eta_abs_mean": 0.008644192479550838, "metrics/particle/matched/delta_phi_abs_mean": 0.009987975470721722, "metrics/particle/matched/delta_r_mean": 0.014782093465328217, "metrics/particle/matched/energy_relative_abs_mean": 0.08688773959875107, "metrics/particle/matched/pid_accuracy": 0.9735216498374939, "metrics/particle/matched/pt_relative_abs_mean": 0.08133339136838913, "metrics/particle/matching/duplicate_fraction": 0.026620080694556236, "metrics/particle/matching/efficiency": 0.8726809024810791, "metrics/particle/matching/f1": 0.8706799745559692, "metrics/particle/matching/purity": 0.8686881065368652, "metrics/particle/matching/target_coverage_dr0p05": 0.8550058603286743, "metrics/particle/matching/target_coverage_dr0p10": 0.9037690162658691, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7669229507446289, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.900509774684906}, "step_train_time": 0.9155728816986084, "step_valid_time": 2.2332645949936705, "step_total_time": 1612.3199560642242, "elapsed_training_loop_seconds": 1612.3199560642242, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003906162849602, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06412370591071703, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514497639919083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9533783783783784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524130948363145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046621621621621624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9514497639919083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533783783783784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524130948363145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514497639919083, "step_test_time": 3.2595160189957824, "step_plot_time": 2.8664025949983625, "step_validation_cycle_time": 8.795698695001192} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..75934b4f2dc0fbecf6c29ceb11a7233c2b4b4a4c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 1.9078054428100586, "Classification": 0.013830224983394146, "Regression_pt": 0.10834281146526337, "Regression_eta": 0.006602664012461901, "Regression_sin_phi": 0.005589616484940052, "Regression_cos_phi": 0.007520891726016998, "Regression_energy": 0.11299091577529907, "Total": 15.353090286254883, "Total_unweighted": 2.162682567257434}, "train_group": {}, "valid": {"Classification_binary": 1.9447752237319946, "Classification": 0.01569853350520134, "Regression_pt": 0.09341080486774445, "Regression_eta": 0.0057974811643362045, "Regression_sin_phi": 0.005243312567472458, "Regression_cos_phi": 0.005375575739890337, "Regression_energy": 0.09276030212640762, "Total": 13.721303939819336, "Total_unweighted": 2.163061233703047, "metrics/particle/class_1/efficiency": 0.9533628225326538, "metrics/particle/class_1/pid_accuracy": 0.9868189096450806, "metrics/particle/class_2/efficiency": 0.5403867959976196, "metrics/particle/class_2/pid_accuracy": 0.8357895016670227, "metrics/particle/class_3/efficiency": 0.8412407040596008, "metrics/particle/class_3/pid_accuracy": 0.9896622896194458, "metrics/particle/class_4/efficiency": 0.8769716024398804, "metrics/particle/class_4/pid_accuracy": 0.6007194519042969, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7681159377098083, "metrics/particle/count/bias_mean": 0.3919999897480011, "metrics/particle/count/mae": 2.799999952316284, "metrics/particle/count/prediction_mean": 96.12000274658203, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.027098555117845535, "metrics/particle/event/energy_response_mean": 1.004953145980835, "metrics/particle/event/met_abs_error": 4.497226238250732, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02795308083295822, "metrics/particle/event/scalar_pt_response_mean": 1.0080195665359497, "metrics/particle/event/vector_pt_closure": 0.02972695417702198, "metrics/particle/matched/delta_eta_abs_mean": 0.008809052407741547, "metrics/particle/matched/delta_phi_abs_mean": 0.009706857614219189, "metrics/particle/matched/delta_r_mean": 0.014657698571681976, "metrics/particle/matched/energy_relative_abs_mean": 0.08643876016139984, "metrics/particle/matched/pid_accuracy": 0.9738452434539795, "metrics/particle/matched/pt_relative_abs_mean": 0.08120531588792801, "metrics/particle/matching/duplicate_fraction": 0.026009155437350273, "metrics/particle/matching/efficiency": 0.8738927245140076, "metrics/particle/matching/f1": 0.8721070885658264, "metrics/particle/matching/purity": 0.8703287839889526, "metrics/particle/matching/target_coverage_dr0p05": 0.854587972164154, "metrics/particle/matching/target_coverage_dr0p10": 0.9039361476898193, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7679675817489624, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9027243852615356}, "step_train_time": 0.8867383003234863, "step_valid_time": 2.002475150002283, "step_total_time": 164.3189766407013, "elapsed_training_loop_seconds": 164.3189766407013, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030449529760215, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06350137462403649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9554956169925826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574324324324325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564630442119474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04256756756756757, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9547297297297297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537630779615255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9554956169925826, "step_test_time": 3.183711787001812, "step_plot_time": 2.750496563996421, "step_validation_cycle_time": 8.335704741999507} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..b5b5e397ac0b888dac404d0354b9ae27d48584a5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 2.3811724185943604, "Classification": 0.01145630981773138, "Regression_pt": 0.08147358149290085, "Regression_eta": 0.0038362767081707716, "Regression_sin_phi": 0.004632963798940182, "Regression_cos_phi": 0.0058573028072714806, "Regression_energy": 0.07940158247947693, "Total": 12.352141380310059, "Total_unweighted": 2.567830435698852}, "train_group": {}, "valid": {"Classification_binary": 2.0197105407714844, "Classification": 0.01645946130156517, "Regression_pt": 0.09913306683301926, "Regression_eta": 0.006146941799670458, "Regression_sin_phi": 0.005368657410144806, "Regression_cos_phi": 0.005833102390170097, "Regression_energy": 0.09887884557247162, "Total": 14.473523139953613, "Total_unweighted": 2.251530616078526, "metrics/particle/class_1/efficiency": 0.9551327228546143, "metrics/particle/class_1/pid_accuracy": 0.9857314825057983, "metrics/particle/class_2/efficiency": 0.5346985459327698, "metrics/particle/class_2/pid_accuracy": 0.8361701965332031, "metrics/particle/class_3/efficiency": 0.8405643105506897, "metrics/particle/class_3/pid_accuracy": 0.989768922328949, "metrics/particle/class_4/efficiency": 0.8548895716667175, "metrics/particle/class_4/pid_accuracy": 0.6346863508224487, "metrics/particle/class_5/efficiency": 0.9903846383094788, "metrics/particle/class_5/pid_accuracy": 0.7475728392601013, "metrics/particle/count/bias_mean": 0.335999995470047, "metrics/particle/count/mae": 2.9760000705718994, "metrics/particle/count/prediction_mean": 96.06400299072266, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.027853339910507202, "metrics/particle/event/energy_response_mean": 1.0038892030715942, "metrics/particle/event/met_abs_error": 4.57973051071167, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027718033641576767, "metrics/particle/event/scalar_pt_response_mean": 1.0056380033493042, "metrics/particle/event/vector_pt_closure": 0.02954758144915104, "metrics/particle/matched/delta_eta_abs_mean": 0.008408146910369396, "metrics/particle/matched/delta_phi_abs_mean": 0.010024262592196465, "metrics/particle/matched/delta_r_mean": 0.014595616608858109, "metrics/particle/matched/energy_relative_abs_mean": 0.08625750988721848, "metrics/particle/matched/pid_accuracy": 0.9737911820411682, "metrics/particle/matched/pt_relative_abs_mean": 0.08061834424734116, "metrics/particle/matching/duplicate_fraction": 0.026232510805130005, "metrics/particle/matching/efficiency": 0.873683750629425, "metrics/particle/matching/f1": 0.8721531629562378, "metrics/particle/matching/purity": 0.8706279397010803, "metrics/particle/matching/target_coverage_dr0p05": 0.8540030121803284, "metrics/particle/matching/target_coverage_dr0p10": 0.9028915166854858, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7700986266136169, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9005515575408936}, "step_train_time": 0.8488473892211914, "step_valid_time": 2.027310254998156, "step_total_time": 1698.4582600593567, "elapsed_training_loop_seconds": 1698.4582600593567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028032639139068, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06391760971957561, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9581081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571380357745528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041891891891891894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551130610867364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.2390981459902832, "step_plot_time": 2.898221536001074, "step_validation_cycle_time": 8.604355907998979} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..0d75ce4f9bd6802b32f37c44941bdbac73b3bfb3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 2.0036041736602783, "Classification": 0.0115877790376544, "Regression_pt": 0.07484087347984314, "Regression_eta": 0.0035993612837046385, "Regression_sin_phi": 0.0023662878666073084, "Regression_cos_phi": 0.0030589322559535503, "Regression_energy": 0.07244081795215607, "Total": 9.764065742492676, "Total_unweighted": 2.1714982255361974}, "train_group": {}, "valid": {"Classification_binary": 2.0185136795043945, "Classification": 0.016310689970850945, "Regression_pt": 0.09944170713424683, "Regression_eta": 0.006073048338294029, "Regression_sin_phi": 0.005358921829611063, "Regression_cos_phi": 0.005914423149079084, "Regression_energy": 0.09889863431453705, "Total": 14.459770202636719, "Total_unweighted": 2.2505111042410135, "metrics/particle/class_1/efficiency": 0.9557521939277649, "metrics/particle/class_1/pid_accuracy": 0.9844444394111633, "metrics/particle/class_2/efficiency": 0.5369738340377808, "metrics/particle/class_2/pid_accuracy": 0.8336864113807678, "metrics/particle/class_3/efficiency": 0.8368924260139465, "metrics/particle/class_3/pid_accuracy": 0.9900704026222229, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.6397058963775635, "metrics/particle/class_5/efficiency": 0.9903846383094788, "metrics/particle/class_5/pid_accuracy": 0.7961165308952332, "metrics/particle/count/bias_mean": 0.6919999718666077, "metrics/particle/count/mae": 2.868000030517578, "metrics/particle/count/prediction_mean": 96.41999816894531, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0291074700653553, "metrics/particle/event/energy_response_mean": 1.0068637132644653, "metrics/particle/event/met_abs_error": 4.586368083953857, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028904857113957405, "metrics/particle/event/scalar_pt_response_mean": 1.0087103843688965, "metrics/particle/event/vector_pt_closure": 0.028048653155565262, "metrics/particle/matched/delta_eta_abs_mean": 0.008832958526909351, "metrics/particle/matched/delta_phi_abs_mean": 0.010145746171474457, "metrics/particle/matched/delta_r_mean": 0.015038208104670048, "metrics/particle/matched/energy_relative_abs_mean": 0.08700286597013474, "metrics/particle/matched/pid_accuracy": 0.9736149311065674, "metrics/particle/matched/pt_relative_abs_mean": 0.08131727576255798, "metrics/particle/matching/duplicate_fraction": 0.02796100452542305, "metrics/particle/matching/efficiency": 0.872597336769104, "metrics/particle/matching/f1": 0.8694548010826111, "metrics/particle/matching/purity": 0.8663347959518433, "metrics/particle/matching/target_coverage_dr0p05": 0.8510780334472656, "metrics/particle/matching/target_coverage_dr0p10": 0.9038525819778442, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7663379311561584, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9004262089729309}, "step_train_time": 0.8613243103027344, "step_valid_time": 2.0049267029971816, "step_total_time": 1782.5046260356903, "elapsed_training_loop_seconds": 1782.5046260356903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005489403108284, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06474514201469261, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9535353535353536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954177897574124, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046464646464646465, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9528619528619529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9535040431266847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548213081591369, "step_test_time": 3.2831621269870084, "step_plot_time": 2.82113448799646, "step_validation_cycle_time": 8.47446807801316} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..53d3af3c40ae5220b8eb46c5495afbc1d27f9ace --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 1.8537503480911255, "Classification": 0.012341596186161041, "Regression_pt": 0.06045105308294296, "Regression_eta": 0.008448371663689613, "Regression_sin_phi": 0.002853801241144538, "Regression_cos_phi": 0.0033286295365542173, "Regression_energy": 0.06345749646425247, "Total": 11.235931396484375, "Total_unweighted": 2.0046312962658703}, "train_group": {}, "valid": {"Classification_binary": 2.0230207443237305, "Classification": 0.01679452322423458, "Regression_pt": 0.09721823036670685, "Regression_eta": 0.006244015879929066, "Regression_sin_phi": 0.005559613928198814, "Regression_cos_phi": 0.005961138289421797, "Regression_energy": 0.09703859686851501, "Total": 14.597372055053711, "Total_unweighted": 2.2518368628807366, "metrics/particle/class_1/efficiency": 0.9550442695617676, "metrics/particle/class_1/pid_accuracy": 0.9844329357147217, "metrics/particle/class_2/efficiency": 0.5307167172431946, "metrics/particle/class_2/pid_accuracy": 0.8370846509933472, "metrics/particle/class_3/efficiency": 0.8390182852745056, "metrics/particle/class_3/pid_accuracy": 0.9905562400817871, "metrics/particle/class_4/efficiency": 0.8548895716667175, "metrics/particle/class_4/pid_accuracy": 0.6236162185668945, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.7756097316741943, "metrics/particle/count/bias_mean": 0.31200000643730164, "metrics/particle/count/mae": 2.8239998817443848, "metrics/particle/count/prediction_mean": 96.04000091552734, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.028534147888422012, "metrics/particle/event/energy_response_mean": 0.9998884797096252, "metrics/particle/event/met_abs_error": 4.5661234855651855, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027399787679314613, "metrics/particle/event/scalar_pt_response_mean": 1.0019949674606323, "metrics/particle/event/vector_pt_closure": 0.029035497456789017, "metrics/particle/matched/delta_eta_abs_mean": 0.008574629202485085, "metrics/particle/matched/delta_phi_abs_mean": 0.010398472659289837, "metrics/particle/matched/delta_r_mean": 0.015028204768896103, "metrics/particle/matched/energy_relative_abs_mean": 0.08620038628578186, "metrics/particle/matched/pid_accuracy": 0.9736640453338623, "metrics/particle/matched/pt_relative_abs_mean": 0.08046716451644897, "metrics/particle/matching/duplicate_fraction": 0.026405664160847664, "metrics/particle/matching/efficiency": 0.8726391196250916, "metrics/particle/matching/f1": 0.8712193965911865, "metrics/particle/matching/purity": 0.8698042631149292, "metrics/particle/matching/target_coverage_dr0p05": 0.8514959216117859, "metrics/particle/matching/target_coverage_dr0p10": 0.901512622833252, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7703075408935547, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.900509774684906}, "step_train_time": 0.8546671867370605, "step_valid_time": 2.2040415449882858, "step_total_time": 1865.9937407970428, "elapsed_training_loop_seconds": 1865.9937407970428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019882537025881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0634804881781319, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9581081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571380357745528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041891891891891894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9567567567567568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557880526493419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.3096077299996978, "step_plot_time": 2.8860710689914413, "step_validation_cycle_time": 8.845335523990798} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..1b1f61442fcb8fc2285b104f69cbe73083d462d6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 1.86223566532135, "Classification": 0.011806285940110683, "Regression_pt": 0.07704711705446243, "Regression_eta": 0.0032465222757309675, "Regression_sin_phi": 0.0025016420986503363, "Regression_cos_phi": 0.0018909932114183903, "Regression_energy": 0.0769682377576828, "Total": 9.279705047607422, "Total_unweighted": 2.0356964636594057}, "train_group": {}, "valid": {"Classification_binary": 2.031125068664551, "Classification": 0.016714949160814285, "Regression_pt": 0.10052112489938736, "Regression_eta": 0.006152131594717503, "Regression_sin_phi": 0.00538741797208786, "Regression_cos_phi": 0.006035208702087402, "Regression_energy": 0.10028476268053055, "Total": 14.658333778381348, "Total_unweighted": 2.2662206636741757, "metrics/particle/class_1/efficiency": 0.9535398483276367, "metrics/particle/class_1/pid_accuracy": 0.9836658835411072, "metrics/particle/class_2/efficiency": 0.5364050269126892, "metrics/particle/class_2/pid_accuracy": 0.8388122916221619, "metrics/particle/class_3/efficiency": 0.8387283682823181, "metrics/particle/class_3/pid_accuracy": 0.9885944724082947, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.625, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7874395847320557, "metrics/particle/count/bias_mean": 0.47600001096725464, "metrics/particle/count/mae": 2.9639999866485596, "metrics/particle/count/prediction_mean": 96.2040023803711, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.030021794140338898, "metrics/particle/event/energy_response_mean": 1.0053397417068481, "metrics/particle/event/met_abs_error": 5.012183666229248, "metrics/particle/event/scalar_pt_relative_abs_error": 0.030084848403930664, "metrics/particle/event/scalar_pt_response_mean": 1.0059857368469238, "metrics/particle/event/vector_pt_closure": 0.030788421630859375, "metrics/particle/matched/delta_eta_abs_mean": 0.008570299483835697, "metrics/particle/matched/delta_phi_abs_mean": 0.010385986417531967, "metrics/particle/matched/delta_r_mean": 0.015026218257844448, "metrics/particle/matched/energy_relative_abs_mean": 0.08706000447273254, "metrics/particle/matched/pid_accuracy": 0.972553551197052, "metrics/particle/matched/pt_relative_abs_mean": 0.08097278326749802, "metrics/particle/matching/duplicate_fraction": 0.027067482471466064, "metrics/particle/matching/efficiency": 0.8723466396331787, "metrics/particle/matching/f1": 0.8701831698417664, "metrics/particle/matching/purity": 0.8680304288864136, "metrics/particle/matching/target_coverage_dr0p05": 0.8518719673156738, "metrics/particle/matching/target_coverage_dr0p10": 0.9027661681175232, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7669229507446289, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9004262089729309}, "step_train_time": 0.922616720199585, "step_valid_time": 2.246801154993591, "step_total_time": 1951.0211203098297, "elapsed_training_loop_seconds": 1951.0211203098297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036834154919818, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06497875677913767, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9575185434929198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568733153638814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957195820694304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0431266846361186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9555256064690026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558476575665655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9575185434929198, "step_test_time": 3.277354093006579, "step_plot_time": 2.780822682994767, "step_validation_cycle_time": 8.680169793995447} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..ff7cf9cb57869eae811979a2c6cbd1f65bd40197 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 2.0186116695404053, "Classification": 0.01021677814424038, "Regression_pt": 0.06710553169250488, "Regression_eta": 0.001753928721882403, "Regression_sin_phi": 0.0022036414593458176, "Regression_cos_phi": 0.002218156587332487, "Regression_energy": 0.06700456887483597, "Total": 8.311164855957031, "Total_unweighted": 2.169114275020547}, "train_group": {}, "valid": {"Classification_binary": 2.0283281803131104, "Classification": 0.016545280814170837, "Regression_pt": 0.10136747360229492, "Regression_eta": 0.00621009198948741, "Regression_sin_phi": 0.00548852514475584, "Regression_cos_phi": 0.006052461452782154, "Regression_energy": 0.10103156417608261, "Total": 14.729310989379883, "Total_unweighted": 2.265023577492684, "metrics/particle/class_1/efficiency": 0.9551327228546143, "metrics/particle/class_1/pid_accuracy": 0.9842490553855896, "metrics/particle/class_2/efficiency": 0.5375426411628723, "metrics/particle/class_2/pid_accuracy": 0.8391534686088562, "metrics/particle/class_3/efficiency": 0.8369891047477722, "metrics/particle/class_3/pid_accuracy": 0.9885707497596741, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.6470588445663452, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.795121967792511, "metrics/particle/count/bias_mean": 0.6439999938011169, "metrics/particle/count/mae": 2.9159998893737793, "metrics/particle/count/prediction_mean": 96.37200164794922, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.029813047498464584, "metrics/particle/event/energy_response_mean": 1.0017085075378418, "metrics/particle/event/met_abs_error": 4.605039119720459, "metrics/particle/event/scalar_pt_relative_abs_error": 0.029895354062318802, "metrics/particle/event/scalar_pt_response_mean": 1.001727819442749, "metrics/particle/event/vector_pt_closure": 0.029519030824303627, "metrics/particle/matched/delta_eta_abs_mean": 0.008947188034653664, "metrics/particle/matched/delta_phi_abs_mean": 0.010440347716212273, "metrics/particle/matched/delta_r_mean": 0.015352264046669006, "metrics/particle/matched/energy_relative_abs_mean": 0.0870763435959816, "metrics/particle/matched/pid_accuracy": 0.9732241034507751, "metrics/particle/matched/pt_relative_abs_mean": 0.08035175502300262, "metrics/particle/matching/duplicate_fraction": 0.026480721309781075, "metrics/particle/matching/efficiency": 0.8723466396331787, "metrics/particle/matching/f1": 0.869422197341919, "metrics/particle/matching/purity": 0.8665172457695007, "metrics/particle/matching/target_coverage_dr0p05": 0.8477352261543274, "metrics/particle/matching/target_coverage_dr0p10": 0.9021393656730652, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7693464756011963, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9008022546768188}, "step_train_time": 0.8934974670410156, "step_valid_time": 2.0175080170010915, "step_total_time": 2038.5922348499298, "elapsed_training_loop_seconds": 2038.5922348499298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009102530898795, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06323756805727565, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1410, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514497639919083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9508086253369272, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9511290866194809, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04919137466307277, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507754551584626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9501347708894878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9504550050556118, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514497639919083, "step_test_time": 3.2582410290051484, "step_plot_time": 2.8703413759940304, "step_validation_cycle_time": 8.590945071991882} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..4a95fcb9d19ba06c63956838245158153e131386 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 1.9508497714996338, "Classification": 0.014156579039990902, "Regression_pt": 0.06719861179590225, "Regression_eta": 0.006239111069589853, "Regression_sin_phi": 0.0018924397882074118, "Regression_cos_phi": 0.003162601962685585, "Regression_energy": 0.06763225048780441, "Total": 10.595508575439453, "Total_unweighted": 2.111131365643814}, "train_group": {}, "valid": {"Classification_binary": 2.043212652206421, "Classification": 0.01689867489039898, "Regression_pt": 0.10057907551527023, "Regression_eta": 0.00626592431217432, "Regression_sin_phi": 0.00536726089194417, "Regression_cos_phi": 0.0060144043527543545, "Regression_energy": 0.10026653856039047, "Total": 14.720438003540039, "Total_unweighted": 2.2786045307293534, "metrics/particle/class_1/efficiency": 0.9544247984886169, "metrics/particle/class_1/pid_accuracy": 0.9841446280479431, "metrics/particle/class_2/efficiency": 0.5284414291381836, "metrics/particle/class_2/pid_accuracy": 0.8363832235336304, "metrics/particle/class_3/efficiency": 0.838051974773407, "metrics/particle/class_3/pid_accuracy": 0.9891617894172668, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.6397058963775635, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.795121967792511, "metrics/particle/count/bias_mean": 0.36399999260902405, "metrics/particle/count/mae": 2.859999895095825, "metrics/particle/count/prediction_mean": 96.09200286865234, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.029405124485492706, "metrics/particle/event/energy_response_mean": 0.9987666606903076, "metrics/particle/event/met_abs_error": 4.2865986824035645, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028681525960564613, "metrics/particle/event/scalar_pt_response_mean": 0.9997050762176514, "metrics/particle/event/vector_pt_closure": 0.028979329392313957, "metrics/particle/matched/delta_eta_abs_mean": 0.008624960668385029, "metrics/particle/matched/delta_phi_abs_mean": 0.010158304125070572, "metrics/particle/matched/delta_r_mean": 0.01488360296934843, "metrics/particle/matched/energy_relative_abs_mean": 0.08623694628477097, "metrics/particle/matched/pid_accuracy": 0.9733033180236816, "metrics/particle/matched/pt_relative_abs_mean": 0.08104900270700455, "metrics/particle/matching/duplicate_fraction": 0.026266494765877724, "metrics/particle/matching/efficiency": 0.8718034625053406, "metrics/particle/matching/f1": 0.8701490759849548, "metrics/particle/matching/purity": 0.8685010075569153, "metrics/particle/matching/target_coverage_dr0p05": 0.8509109020233154, "metrics/particle/matching/target_coverage_dr0p10": 0.9019304513931274, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7662543654441833, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8991308808326721}, "step_train_time": 0.9972760677337646, "step_valid_time": 2.1795705359982094, "step_total_time": 2123.4613683223724, "elapsed_training_loop_seconds": 2123.4613683223724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991333181038246, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06618132958387146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9555256064690026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9558476575665655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0444743935309973, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9554956169925826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548517520215634, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551735760026964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.31577380199451, "step_plot_time": 3.077613964007469, "step_validation_cycle_time": 9.065155009011505} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..b942c2668e8bb0ab6e348096202929b6384a08ea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 1.9004831314086914, "Classification": 0.01165676862001419, "Regression_pt": 0.06695079058408737, "Regression_eta": 0.0016037706518545747, "Regression_sin_phi": 0.002188875572755933, "Regression_cos_phi": 0.003018934978172183, "Regression_energy": 0.06490927189588547, "Total": 8.57507038116455, "Total_unweighted": 2.050811543711461}, "train_group": {}, "valid": {"Classification_binary": 2.0476479530334473, "Classification": 0.01719815842807293, "Regression_pt": 0.10117392987012863, "Regression_eta": 0.006322339177131653, "Regression_sin_phi": 0.005507666617631912, "Regression_cos_phi": 0.006089709233492613, "Regression_energy": 0.10087600350379944, "Total": 14.887961387634277, "Total_unweighted": 2.2848157598637044, "metrics/particle/class_1/efficiency": 0.9556637406349182, "metrics/particle/class_1/pid_accuracy": 0.9846282005310059, "metrics/particle/class_2/efficiency": 0.5352673530578613, "metrics/particle/class_2/pid_accuracy": 0.8331562280654907, "metrics/particle/class_3/efficiency": 0.8323509693145752, "metrics/particle/class_3/pid_accuracy": 0.9903645515441895, "metrics/particle/class_4/efficiency": 0.8675078749656677, "metrics/particle/class_4/pid_accuracy": 0.6290909051895142, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.7707316875457764, "metrics/particle/count/bias_mean": 0.25999999046325684, "metrics/particle/count/mae": 3.0360000133514404, "metrics/particle/count/prediction_mean": 95.98799896240234, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02963194064795971, "metrics/particle/event/energy_response_mean": 1.0061051845550537, "metrics/particle/event/met_abs_error": 4.670956611633301, "metrics/particle/event/scalar_pt_relative_abs_error": 0.029746167361736298, "metrics/particle/event/scalar_pt_response_mean": 1.007978081703186, "metrics/particle/event/vector_pt_closure": 0.029322151094675064, "metrics/particle/matched/delta_eta_abs_mean": 0.008860601112246513, "metrics/particle/matched/delta_phi_abs_mean": 0.010322273708879948, "metrics/particle/matched/delta_r_mean": 0.01521927211433649, "metrics/particle/matched/energy_relative_abs_mean": 0.08612997084856033, "metrics/particle/matched/pid_accuracy": 0.9733608365058899, "metrics/particle/matched/pt_relative_abs_mean": 0.08072800189256668, "metrics/particle/matching/duplicate_fraction": 0.027295079082250595, "metrics/particle/matching/efficiency": 0.8705499172210693, "metrics/particle/matching/f1": 0.8693692684173584, "metrics/particle/matching/purity": 0.8681918382644653, "metrics/particle/matching/target_coverage_dr0p05": 0.8466905951499939, "metrics/particle/matching/target_coverage_dr0p10": 0.9002172946929932, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.766379714012146, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9003844261169434}, "step_train_time": 0.8657407760620117, "step_valid_time": 2.04555781801173, "step_total_time": 2212.5293352603912, "elapsed_training_loop_seconds": 2212.5293352603912, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043221846978014, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06495324263656754, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1410, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1408, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507754551584626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9488559892328399, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498147524418996, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05114401076716016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494268374915711, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9475100942126514, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9484674974738969, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507754551584626, "step_test_time": 3.215411259996472, "step_plot_time": 2.925281803996768, "step_validation_cycle_time": 8.637496303999797} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..f14e2ebbb8c0dcc7f85b852767a959368fb0b5ae --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 2.316429376602173, "Classification": 0.013127686455845833, "Regression_pt": 0.09043899178504944, "Regression_eta": 0.002066314220428467, "Regression_sin_phi": 0.00283723883330822, "Regression_cos_phi": 0.0027885029558092356, "Regression_energy": 0.08838686347007751, "Total": 10.44548511505127, "Total_unweighted": 2.5160749743226916}, "train_group": {}, "valid": {"Classification_binary": 2.052908182144165, "Classification": 0.01739712804555893, "Regression_pt": 0.10249137878417969, "Regression_eta": 0.006358694285154343, "Regression_sin_phi": 0.005541547201573849, "Regression_cos_phi": 0.006114586256444454, "Regression_energy": 0.10202482342720032, "Total": 15.003480911254883, "Total_unweighted": 2.2928363401442766, "metrics/particle/class_1/efficiency": 0.9551327228546143, "metrics/particle/class_1/pid_accuracy": 0.9813768267631531, "metrics/particle/class_2/efficiency": 0.532423198223114, "metrics/particle/class_2/pid_accuracy": 0.8440170884132385, "metrics/particle/class_3/efficiency": 0.83563631772995, "metrics/particle/class_3/pid_accuracy": 0.9875115752220154, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.658088207244873, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.795121967792511, "metrics/particle/count/bias_mean": 0.35600000619888306, "metrics/particle/count/mae": 2.9079999923706055, "metrics/particle/count/prediction_mean": 96.08399963378906, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.03091202676296234, "metrics/particle/event/energy_response_mean": 1.0058293342590332, "metrics/particle/event/met_abs_error": 4.922903537750244, "metrics/particle/event/scalar_pt_relative_abs_error": 0.030401350930333138, "metrics/particle/event/scalar_pt_response_mean": 1.007951259613037, "metrics/particle/event/vector_pt_closure": 0.030710341408848763, "metrics/particle/matched/delta_eta_abs_mean": 0.008762490935623646, "metrics/particle/matched/delta_phi_abs_mean": 0.010440480895340443, "metrics/particle/matched/delta_r_mean": 0.01521485485136509, "metrics/particle/matched/energy_relative_abs_mean": 0.08613181114196777, "metrics/particle/matched/pid_accuracy": 0.9717080593109131, "metrics/particle/matched/pt_relative_abs_mean": 0.08119438588619232, "metrics/particle/matching/duplicate_fraction": 0.02735106833279133, "metrics/particle/matching/efficiency": 0.8713855743408203, "metrics/particle/matching/f1": 0.8697683215141296, "metrics/particle/matching/purity": 0.8681570291519165, "metrics/particle/matching/target_coverage_dr0p05": 0.8481113314628601, "metrics/particle/matching/target_coverage_dr0p10": 0.9015544056892395, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7652515172958374, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8982115983963013}, "step_train_time": 0.9063611030578613, "step_valid_time": 2.1560347189952154, "step_total_time": 2297.7784101963043, "elapsed_training_loop_seconds": 2297.7784101963043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0037388609151003, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06465238046354005, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.952124072825354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514824797843666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9518031681833502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04851752021563342, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9487525286581254, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9481132075471698, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948432760364004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.952124072825354, "step_test_time": 3.5799520239961566, "step_plot_time": 3.4241515250032535, "step_validation_cycle_time": 9.650860195004498} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..440dbe976696e4fe576f0483b521e4dbb5855065 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 2.231506109237671, "Classification": 0.012289362028241158, "Regression_pt": 0.08346650749444962, "Regression_eta": 0.005093209445476532, "Regression_sin_phi": 0.004064705688506365, "Regression_cos_phi": 0.004681567661464214, "Regression_energy": 0.08147475868463516, "Total": 12.193061828613281, "Total_unweighted": 2.422576220240444}, "train_group": {}, "valid": {"Classification_binary": 2.060403347015381, "Classification": 0.017409618943929672, "Regression_pt": 0.10305362939834595, "Regression_eta": 0.006526938173919916, "Regression_sin_phi": 0.00550307659432292, "Regression_cos_phi": 0.00610731914639473, "Regression_energy": 0.10281290858983994, "Total": 15.082381248474121, "Total_unweighted": 2.301816837862134, "metrics/particle/class_1/efficiency": 0.9551327228546143, "metrics/particle/class_1/pid_accuracy": 0.9823960065841675, "metrics/particle/class_2/efficiency": 0.5290102362632751, "metrics/particle/class_2/pid_accuracy": 0.8215053677558899, "metrics/particle/class_3/efficiency": 0.8336071372032166, "metrics/particle/class_3/pid_accuracy": 0.9900313019752502, "metrics/particle/class_4/efficiency": 0.8675078749656677, "metrics/particle/class_4/pid_accuracy": 0.6399999856948853, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.8097561001777649, "metrics/particle/count/bias_mean": 0.5360000133514404, "metrics/particle/count/mae": 2.9600000381469727, "metrics/particle/count/prediction_mean": 96.26399993896484, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.030899113044142723, "metrics/particle/event/energy_response_mean": 1.0028071403503418, "metrics/particle/event/met_abs_error": 4.649714469909668, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031128618866205215, "metrics/particle/event/scalar_pt_response_mean": 1.0045866966247559, "metrics/particle/event/vector_pt_closure": 0.03051251545548439, "metrics/particle/matched/delta_eta_abs_mean": 0.00885178055614233, "metrics/particle/matched/delta_phi_abs_mean": 0.01009515579789877, "metrics/particle/matched/delta_r_mean": 0.015018638223409653, "metrics/particle/matched/energy_relative_abs_mean": 0.08595087379217148, "metrics/particle/matched/pid_accuracy": 0.9721555709838867, "metrics/particle/matched/pt_relative_abs_mean": 0.080687515437603, "metrics/particle/matching/duplicate_fraction": 0.027757002040743828, "metrics/particle/matching/efficiency": 0.8703827261924744, "metrics/particle/matching/f1": 0.867952823638916, "metrics/particle/matching/purity": 0.8655364513397217, "metrics/particle/matching/target_coverage_dr0p05": 0.8471920490264893, "metrics/particle/matching/target_coverage_dr0p10": 0.9002590775489807, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7655022740364075, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8985876441001892}, "step_train_time": 0.8331501483917236, "step_valid_time": 2.0033093230013037, "step_total_time": 2386.831665992737, "elapsed_training_loop_seconds": 2386.831665992737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020474276738989, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06721391032421731, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9515803631472763, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528619528619529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0484196368527236, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9502353732347008, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9515151515151515, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9541469993256911, "step_test_time": 3.2508476009970764, "step_plot_time": 2.8835726950055687, "step_validation_cycle_time": 8.575459328989382} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..f55d4d22e136ff69845f95b80958b0f39bc47b34 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 1.8266229629516602, "Classification": 0.011980791576206684, "Regression_pt": 0.07849196344614029, "Regression_eta": 0.0024836831726133823, "Regression_sin_phi": 0.0032163788564503193, "Regression_cos_phi": 0.003121713176369667, "Regression_energy": 0.07852625101804733, "Total": 9.776959419250488, "Total_unweighted": 2.004443744197488}, "train_group": {}, "valid": {"Classification_binary": 2.0667827129364014, "Classification": 0.017813216894865036, "Regression_pt": 0.10369011759757996, "Regression_eta": 0.006493609864264727, "Regression_sin_phi": 0.005587623454630375, "Regression_cos_phi": 0.006262089591473341, "Regression_energy": 0.10348157584667206, "Total": 15.244763374328613, "Total_unweighted": 2.310110946185887, "metrics/particle/class_1/efficiency": 0.9552212357521057, "metrics/particle/class_1/pid_accuracy": 0.9842504858970642, "metrics/particle/class_2/efficiency": 0.5295790433883667, "metrics/particle/class_2/pid_accuracy": 0.8270676732063293, "metrics/particle/class_3/efficiency": 0.8361194133758545, "metrics/particle/class_3/pid_accuracy": 0.9879810214042664, "metrics/particle/class_4/efficiency": 0.85173499584198, "metrics/particle/class_4/pid_accuracy": 0.6407407522201538, "metrics/particle/class_5/efficiency": 0.9807692170143127, "metrics/particle/class_5/pid_accuracy": 0.7549019455909729, "metrics/particle/count/bias_mean": 0.7960000038146973, "metrics/particle/count/mae": 3.0360000133514404, "metrics/particle/count/prediction_mean": 96.52400207519531, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0295106191188097, "metrics/particle/event/energy_response_mean": 1.0039867162704468, "metrics/particle/event/met_abs_error": 4.635102272033691, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02949271909892559, "metrics/particle/event/scalar_pt_response_mean": 1.0054649114608765, "metrics/particle/event/vector_pt_closure": 0.030460599809885025, "metrics/particle/matched/delta_eta_abs_mean": 0.009384503588080406, "metrics/particle/matched/delta_phi_abs_mean": 0.010432995855808258, "metrics/particle/matched/delta_r_mean": 0.01567203365266323, "metrics/particle/matched/energy_relative_abs_mean": 0.08625774085521698, "metrics/particle/matched/pid_accuracy": 0.9720889925956726, "metrics/particle/matched/pt_relative_abs_mean": 0.08073791116476059, "metrics/particle/matching/duplicate_fraction": 0.027765115723013878, "metrics/particle/matching/efficiency": 0.8713020086288452, "metrics/particle/matching/f1": 0.8676944971084595, "metrics/particle/matching/purity": 0.8641166687011719, "metrics/particle/matching/target_coverage_dr0p05": 0.8465652465820312, "metrics/particle/matching/target_coverage_dr0p10": 0.8998411893844604, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7672154307365417, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9003426432609558}, "step_train_time": 0.9752645492553711, "step_valid_time": 2.2486281799938297, "step_total_time": 2473.6496324539185, "elapsed_training_loop_seconds": 2473.6496324539185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015911274827536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06624832257449276, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9496644295302014, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9519004372687521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050335570469798654, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9483221476510068, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9505549949545913, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9541469993256911, "step_test_time": 3.7539865440048743, "step_plot_time": 2.897526316999574, "step_validation_cycle_time": 9.344058916001813} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..5af30c5ee7e64e4dfbcbe417897476bed98f3ab6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 1.8640536069869995, "Classification": 0.012433301657438278, "Regression_pt": 0.06929157674312592, "Regression_eta": 0.003456115024164319, "Regression_sin_phi": 0.004298605490475893, "Regression_cos_phi": 0.0047810315154492855, "Regression_energy": 0.07014361023902893, "Total": 10.87678337097168, "Total_unweighted": 2.028457847656682}, "train_group": {}, "valid": {"Classification_binary": 1.949455738067627, "Classification": 0.015771035104990005, "Regression_pt": 0.09276320785284042, "Regression_eta": 0.005761976819485426, "Regression_sin_phi": 0.0052418699488043785, "Regression_cos_phi": 0.005389938596636057, "Regression_energy": 0.09207872301340103, "Total": 13.701034545898438, "Total_unweighted": 2.1664624894037843, "metrics/particle/class_1/efficiency": 0.9550442695617676, "metrics/particle/class_1/pid_accuracy": 0.985822856426239, "metrics/particle/class_2/efficiency": 0.5472127199172974, "metrics/particle/class_2/pid_accuracy": 0.8347193598747253, "metrics/particle/class_3/efficiency": 0.8446226716041565, "metrics/particle/class_3/pid_accuracy": 0.9899324774742126, "metrics/particle/class_4/efficiency": 0.8738170266151428, "metrics/particle/class_4/pid_accuracy": 0.5956678986549377, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7632850408554077, "metrics/particle/count/bias_mean": 0.6480000019073486, "metrics/particle/count/mae": 2.9600000381469727, "metrics/particle/count/prediction_mean": 96.3759994506836, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02722783200442791, "metrics/particle/event/energy_response_mean": 1.0058884620666504, "metrics/particle/event/met_abs_error": 4.425021648406982, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02776699885725975, "metrics/particle/event/scalar_pt_response_mean": 1.0086336135864258, "metrics/particle/event/vector_pt_closure": 0.028857462108135223, "metrics/particle/matched/delta_eta_abs_mean": 0.008329888805747032, "metrics/particle/matched/delta_phi_abs_mean": 0.009676129557192326, "metrics/particle/matched/delta_r_mean": 0.01428235787898302, "metrics/particle/matched/energy_relative_abs_mean": 0.08619745820760727, "metrics/particle/matched/pid_accuracy": 0.9732589721679688, "metrics/particle/matched/pt_relative_abs_mean": 0.08078744262456894, "metrics/particle/matching/duplicate_fraction": 0.02573254704475403, "metrics/particle/matching/efficiency": 0.8766087293624878, "metrics/particle/matching/f1": 0.8736517429351807, "metrics/particle/matching/purity": 0.8707147240638733, "metrics/particle/matching/target_coverage_dr0p05": 0.8588500618934631, "metrics/particle/matching/target_coverage_dr0p10": 0.9061925411224365, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7706000208854675, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9031004309654236}, "step_train_time": 0.8770577907562256, "step_valid_time": 2.342600730000413, "step_total_time": 246.7717638015747, "elapsed_training_loop_seconds": 246.7717638015747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036024588942076, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060201032851808445, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1479, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9581928523263655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9607843137254902, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9594868332207968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0392156862745098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9587559161595672, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574611748818366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9581928523263655, "step_test_time": 3.2414873390080174, "step_plot_time": 2.83651729299163, "step_validation_cycle_time": 8.882958640999277} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..baa11ec4fd2dcf9e6d88fd2f102b794bc4b7df00 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 2.208850145339966, "Classification": 0.01354360394179821, "Regression_pt": 0.06685349345207214, "Regression_eta": 0.0028658683877438307, "Regression_sin_phi": 0.002438400639221072, "Regression_cos_phi": 0.003179872641339898, "Regression_energy": 0.06541456282138824, "Total": 9.737038612365723, "Total_unweighted": 2.363145947223529}, "train_group": {}, "valid": {"Classification_binary": 2.073180913925171, "Classification": 0.017968645319342613, "Regression_pt": 0.10404114425182343, "Regression_eta": 0.006528014782816172, "Regression_sin_phi": 0.005715896375477314, "Regression_cos_phi": 0.006289257667958736, "Regression_energy": 0.10401908308267593, "Total": 15.358911514282227, "Total_unweighted": 2.317742955405265, "metrics/particle/class_1/efficiency": 0.9536283016204834, "metrics/particle/class_1/pid_accuracy": 0.9839457869529724, "metrics/particle/class_2/efficiency": 0.5199089646339417, "metrics/particle/class_2/pid_accuracy": 0.8260393738746643, "metrics/particle/class_3/efficiency": 0.8260701298713684, "metrics/particle/class_3/pid_accuracy": 0.9890045523643494, "metrics/particle/class_4/efficiency": 0.8675078749656677, "metrics/particle/class_4/pid_accuracy": 0.6218181848526001, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.8048780560493469, "metrics/particle/count/bias_mean": -0.06800000369548798, "metrics/particle/count/mae": 3.059999942779541, "metrics/particle/count/prediction_mean": 95.66000366210938, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.03090299665927887, "metrics/particle/event/energy_response_mean": 1.0083608627319336, "metrics/particle/event/met_abs_error": 5.0431227684021, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03151305764913559, "metrics/particle/event/scalar_pt_response_mean": 1.0088462829589844, "metrics/particle/event/vector_pt_closure": 0.032162535935640335, "metrics/particle/matched/delta_eta_abs_mean": 0.008680449798703194, "metrics/particle/matched/delta_phi_abs_mean": 0.010675247758626938, "metrics/particle/matched/delta_r_mean": 0.015352515503764153, "metrics/particle/matched/energy_relative_abs_mean": 0.08849158138036728, "metrics/particle/matched/pid_accuracy": 0.9724889993667603, "metrics/particle/matched/pt_relative_abs_mean": 0.08169710636138916, "metrics/particle/matching/duplicate_fraction": 0.027305038645863533, "metrics/particle/matching/efficiency": 0.8657445907592773, "metrics/particle/matching/f1": 0.8660522103309631, "metrics/particle/matching/purity": 0.8663600087165833, "metrics/particle/matching/target_coverage_dr0p05": 0.8415928483009338, "metrics/particle/matching/target_coverage_dr0p10": 0.8971251845359802, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7599866390228271, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8939495086669922}, "step_train_time": 0.9407782554626465, "step_valid_time": 2.440649244002998, "step_total_time": 2560.7158403396606, "elapsed_training_loop_seconds": 2560.7158403396606, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067518244917721, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06915322253731693, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1409, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1408, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9501011463250169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9475453934095495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9488215488215488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05245460659045057, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494268374915711, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9468728984532616, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9481481481481482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9501011463250169, "step_test_time": 3.3345644640066894, "step_plot_time": 2.9094068089907523, "step_validation_cycle_time": 9.130875995993847} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..27c64fb651f711abbbb1e49e0d50f9a80000ffc8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 2.0693299770355225, "Classification": 0.01146741397678852, "Regression_pt": 0.08792687952518463, "Regression_eta": 0.004188376944512129, "Regression_sin_phi": 0.004442910198122263, "Regression_cos_phi": 0.0028327093459665775, "Regression_energy": 0.08474082499742508, "Total": 11.212651252746582, "Total_unweighted": 2.2649290920235217}, "train_group": {}, "valid": {"Classification_binary": 2.068474054336548, "Classification": 0.018351899459958076, "Regression_pt": 0.10371090471744537, "Regression_eta": 0.006618593819439411, "Regression_sin_phi": 0.005576761439442635, "Regression_cos_phi": 0.006097477860748768, "Regression_energy": 0.1036713644862175, "Total": 15.300716400146484, "Total_unweighted": 2.3125010561197996, "metrics/particle/class_1/efficiency": 0.9566371440887451, "metrics/particle/class_1/pid_accuracy": 0.9823311567306519, "metrics/particle/class_2/efficiency": 0.5170648694038391, "metrics/particle/class_2/pid_accuracy": 0.8305830359458923, "metrics/particle/class_3/efficiency": 0.8321576714515686, "metrics/particle/class_3/pid_accuracy": 0.989549458026886, "metrics/particle/class_4/efficiency": 0.8738170266151428, "metrics/particle/class_4/pid_accuracy": 0.6245487332344055, "metrics/particle/class_5/efficiency": 0.9807692170143127, "metrics/particle/class_5/pid_accuracy": 0.7352941036224365, "metrics/particle/count/bias_mean": 0.3199999928474426, "metrics/particle/count/mae": 2.888000011444092, "metrics/particle/count/prediction_mean": 96.0479965209961, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.030240023508667946, "metrics/particle/event/energy_response_mean": 1.0050692558288574, "metrics/particle/event/met_abs_error": 4.738057613372803, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03110538423061371, "metrics/particle/event/scalar_pt_response_mean": 1.007483720779419, "metrics/particle/event/vector_pt_closure": 0.03046543523669243, "metrics/particle/matched/delta_eta_abs_mean": 0.008775129914283752, "metrics/particle/matched/delta_phi_abs_mean": 0.010363328270614147, "metrics/particle/matched/delta_r_mean": 0.015152870677411556, "metrics/particle/matched/energy_relative_abs_mean": 0.08705510944128036, "metrics/particle/matched/pid_accuracy": 0.9715068340301514, "metrics/particle/matched/pt_relative_abs_mean": 0.08131179213523865, "metrics/particle/matching/duplicate_fraction": 0.027028152719140053, "metrics/particle/matching/efficiency": 0.8696306347846985, "metrics/particle/matching/f1": 0.8681795597076416, "metrics/particle/matching/purity": 0.8667333126068115, "metrics/particle/matching/target_coverage_dr0p05": 0.8485291600227356, "metrics/particle/matching/target_coverage_dr0p10": 0.9004679918289185, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7626190781593323, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8967491388320923}, "step_train_time": 0.9316143989562988, "step_valid_time": 2.2690135470038513, "step_total_time": 2646.130401611328, "elapsed_training_loop_seconds": 2646.130401611328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054812896741956, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06739801170841353, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9554956169925826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9522849462365591, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9538875799394143, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04771505376344086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495967741935484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511948838774823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9554956169925826, "step_test_time": 3.3047078259987757, "step_plot_time": 2.904813771994668, "step_validation_cycle_time": 8.937631136999698} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..4ccc3421bc686924f30ad453f42f1a012a8fffc1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 56.93950177935943, "train": {"Classification_binary": 1.9677079916000366, "Classification": 0.011301152408123016, "Regression_pt": 0.08127564191818237, "Regression_eta": 0.00196000374853611, "Regression_sin_phi": 0.002221871167421341, "Regression_cos_phi": 0.0016710502095520496, "Regression_energy": 0.08485352247953415, "Total": 8.924121856689453, "Total_unweighted": 2.1509912335313857}, "train_group": {}, "valid": {"Classification_binary": 2.080263376235962, "Classification": 0.018515590578317642, "Regression_pt": 0.10495661199092865, "Regression_eta": 0.00670994259417057, "Regression_sin_phi": 0.005901744589209557, "Regression_cos_phi": 0.006315039936453104, "Regression_energy": 0.10489322245121002, "Total": 15.616089820861816, "Total_unweighted": 2.3275555283762515, "metrics/particle/class_1/efficiency": 0.9561946988105774, "metrics/particle/class_1/pid_accuracy": 0.9822304248809814, "metrics/particle/class_2/efficiency": 0.5210466384887695, "metrics/particle/class_2/pid_accuracy": 0.8395196795463562, "metrics/particle/class_3/efficiency": 0.8343801498413086, "metrics/particle/class_3/pid_accuracy": 0.9886508584022522, "metrics/particle/class_4/efficiency": 0.8832807540893555, "metrics/particle/class_4/pid_accuracy": 0.6357142925262451, "metrics/particle/class_5/efficiency": 0.9855769276618958, "metrics/particle/class_5/pid_accuracy": 0.7414634227752686, "metrics/particle/count/bias_mean": 0.5479999780654907, "metrics/particle/count/mae": 2.931999921798706, "metrics/particle/count/prediction_mean": 96.2760009765625, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.03119376301765442, "metrics/particle/event/energy_response_mean": 1.0078037977218628, "metrics/particle/event/met_abs_error": 5.039955139160156, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031097855418920517, "metrics/particle/event/scalar_pt_response_mean": 1.0090254545211792, "metrics/particle/event/vector_pt_closure": 0.03180261701345444, "metrics/particle/matched/delta_eta_abs_mean": 0.009401002898812294, "metrics/particle/matched/delta_phi_abs_mean": 0.010460011661052704, "metrics/particle/matched/delta_r_mean": 0.015715474262833595, "metrics/particle/matched/energy_relative_abs_mean": 0.08631564676761627, "metrics/particle/matched/pid_accuracy": 0.9715944528579712, "metrics/particle/matched/pt_relative_abs_mean": 0.0804104432463646, "metrics/particle/matching/duplicate_fraction": 0.0267564095556736, "metrics/particle/matching/efficiency": 0.8708423972129822, "metrics/particle/matching/f1": 0.8683568835258484, "metrics/particle/matching/purity": 0.8658855557441711, "metrics/particle/matching/target_coverage_dr0p05": 0.84531170129776, "metrics/particle/matching/target_coverage_dr0p10": 0.9010947942733765, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7677586674690247, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8986712098121643}, "step_train_time": 0.8909683227539062, "step_valid_time": 2.345286447991384, "step_total_time": 2731.743241786957, "elapsed_training_loop_seconds": 2731.743241786957, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032124249229917, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07050637750263244, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1408, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494268374915711, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9500674763832658, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9497470489038786, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04993252361673414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.947403910991234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480431848852902, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9477234401349073, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494268374915711, "step_test_time": 3.451897064995137, "step_plot_time": 3.027706357999705, "step_validation_cycle_time": 9.400877792999381} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_33000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..a6e8d0f858cef952543ef4dd24ac93541d17f199 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 58.718861209964416, "train": {"Classification_binary": 1.8377790451049805, "Classification": 0.008798014372587204, "Regression_pt": 0.06306784600019455, "Regression_eta": 0.0008226842619478703, "Regression_sin_phi": 0.001487730536609888, "Regression_cos_phi": 0.002229090081527829, "Regression_energy": 0.06226034089922905, "Total": 7.178787708282471, "Total_unweighted": 1.9764447512570769}, "train_group": {}, "valid": {"Classification_binary": 2.0771689414978027, "Classification": 0.01903662085533142, "Regression_pt": 0.10541233420372009, "Regression_eta": 0.006710276938974857, "Regression_sin_phi": 0.005661453120410442, "Regression_cos_phi": 0.006368011236190796, "Regression_energy": 0.10536500811576843, "Total": 15.631499290466309, "Total_unweighted": 2.3257226459681988, "metrics/particle/class_1/efficiency": 0.9534513354301453, "metrics/particle/class_1/pid_accuracy": 0.9821793437004089, "metrics/particle/class_2/efficiency": 0.5153583884239197, "metrics/particle/class_2/pid_accuracy": 0.8311258554458618, "metrics/particle/class_3/efficiency": 0.8340902328491211, "metrics/particle/class_3/pid_accuracy": 0.9887627363204956, "metrics/particle/class_4/efficiency": 0.8611987233161926, "metrics/particle/class_4/pid_accuracy": 0.6336996555328369, "metrics/particle/class_5/efficiency": 0.9807692170143127, "metrics/particle/class_5/pid_accuracy": 0.7598039507865906, "metrics/particle/count/bias_mean": 0.4440000057220459, "metrics/particle/count/mae": 3.0840001106262207, "metrics/particle/count/prediction_mean": 96.1719970703125, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.032166190445423126, "metrics/particle/event/energy_response_mean": 1.009487271308899, "metrics/particle/event/met_abs_error": 5.257565498352051, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03157992660999298, "metrics/particle/event/scalar_pt_response_mean": 1.0102555751800537, "metrics/particle/event/vector_pt_closure": 0.033436983823776245, "metrics/particle/matched/delta_eta_abs_mean": 0.008922315202653408, "metrics/particle/matched/delta_phi_abs_mean": 0.010610299184918404, "metrics/particle/matched/delta_r_mean": 0.015452156774699688, "metrics/particle/matched/energy_relative_abs_mean": 0.08711512386798859, "metrics/particle/matched/pid_accuracy": 0.9715715050697327, "metrics/particle/matched/pt_relative_abs_mean": 0.08073829114437103, "metrics/particle/matching/duplicate_fraction": 0.028449028730392456, "metrics/particle/matching/efficiency": 0.8686695694923401, "metrics/particle/matching/f1": 0.8666597008705139, "metrics/particle/matching/purity": 0.8646591305732727, "metrics/particle/matching/target_coverage_dr0p05": 0.8454788327217102, "metrics/particle/matching/target_coverage_dr0p10": 0.8996740579605103, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7640397548675537, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8968327045440674}, "step_train_time": 0.882352352142334, "step_valid_time": 2.070125279991771, "step_total_time": 2819.081733942032, "elapsed_training_loop_seconds": 2819.081733942032, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004611066440559, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0682479829997863, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1408, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494268374915711, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.946236559139785, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9478290138000673, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053763440860215055, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.947403910991234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9442204301075269, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9458094917536183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494268374915711, "step_test_time": 3.2795023109938484, "step_plot_time": 2.9202097230008803, "step_validation_cycle_time": 8.727590202004649} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..977d583c74e7c61f79de228b753f5ac2a8524868 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 2.167673349380493, "Classification": 0.018224168568849564, "Regression_pt": 0.11295200884342194, "Regression_eta": 0.00420526135712862, "Regression_sin_phi": 0.005437053274363279, "Regression_cos_phi": 0.009967234916985035, "Regression_energy": 0.11367249488830566, "Total": 16.30008888244629, "Total_unweighted": 2.4321315712295473}, "train_group": {}, "valid": {"Classification_binary": 1.9488317966461182, "Classification": 0.015797533094882965, "Regression_pt": 0.09298616647720337, "Regression_eta": 0.0057871476747095585, "Regression_sin_phi": 0.005275615956634283, "Regression_cos_phi": 0.005432772450149059, "Regression_energy": 0.09226550161838531, "Total": 13.749306678771973, "Total_unweighted": 2.1663765339180827, "metrics/particle/class_1/efficiency": 0.9545132517814636, "metrics/particle/class_1/pid_accuracy": 0.9860003590583801, "metrics/particle/class_2/efficiency": 0.5500568747520447, "metrics/particle/class_2/pid_accuracy": 0.8376421928405762, "metrics/particle/class_3/efficiency": 0.8437530398368835, "metrics/particle/class_3/pid_accuracy": 0.989234983921051, "metrics/particle/class_4/efficiency": 0.8643532991409302, "metrics/particle/class_4/pid_accuracy": 0.6058394312858582, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7584540843963623, "metrics/particle/count/bias_mean": 0.6399999856948853, "metrics/particle/count/mae": 2.8480000495910645, "metrics/particle/count/prediction_mean": 96.36799621582031, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.025841886177659035, "metrics/particle/event/energy_response_mean": 1.0042688846588135, "metrics/particle/event/met_abs_error": 4.200661659240723, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027031783014535904, "metrics/particle/event/scalar_pt_response_mean": 1.0064678192138672, "metrics/particle/event/vector_pt_closure": 0.02852599322795868, "metrics/particle/matched/delta_eta_abs_mean": 0.008295352570712566, "metrics/particle/matched/delta_phi_abs_mean": 0.009655185043811798, "metrics/particle/matched/delta_r_mean": 0.014235997572541237, "metrics/particle/matched/energy_relative_abs_mean": 0.08694621175527573, "metrics/particle/matched/pid_accuracy": 0.9732900857925415, "metrics/particle/matched/pt_relative_abs_mean": 0.08114002645015717, "metrics/particle/matching/duplicate_fraction": 0.02660634182393551, "metrics/particle/matching/efficiency": 0.8760654926300049, "metrics/particle/matching/f1": 0.8731467723846436, "metrics/particle/matching/purity": 0.8702473640441895, "metrics/particle/matching/target_coverage_dr0p05": 0.857220470905304, "metrics/particle/matching/target_coverage_dr0p10": 0.9056493639945984, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7703911066055298, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9032257795333862}, "step_train_time": 0.874772310256958, "step_valid_time": 2.0225196899991715, "step_total_time": 330.7906267642975, "elapsed_training_loop_seconds": 330.7906267642975, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029692940724186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06002637009834211, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588671611598112, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9595141700404858, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591905564924115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04048582995951417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956844234659474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574898785425101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571669477234401, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588671611598112, "step_test_time": 3.365393480999046, "step_plot_time": 2.9985807280027075, "step_validation_cycle_time": 8.945799833993078} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..0c08e105eb176639a671dd1930459aba5c39a08a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 1.9343807697296143, "Classification": 0.01904117874801159, "Regression_pt": 0.10312822461128235, "Regression_eta": 0.007448762189596891, "Regression_sin_phi": 0.0042983004823327065, "Regression_cos_phi": 0.003583471290767193, "Regression_energy": 0.10276859998703003, "Total": 14.11878490447998, "Total_unweighted": 2.174649307038635}, "train_group": {}, "valid": {"Classification_binary": 1.9525070190429688, "Classification": 0.015774240717291832, "Regression_pt": 0.09316114336252213, "Regression_eta": 0.005819158628582954, "Regression_sin_phi": 0.0052946568466722965, "Regression_cos_phi": 0.005403386894613504, "Regression_energy": 0.09261868894100189, "Total": 13.767990112304688, "Total_unweighted": 2.1705782944336534, "metrics/particle/class_1/efficiency": 0.9538053274154663, "metrics/particle/class_1/pid_accuracy": 0.9863610863685608, "metrics/particle/class_2/efficiency": 0.5443686246871948, "metrics/particle/class_2/pid_accuracy": 0.8422152400016785, "metrics/particle/class_3/efficiency": 0.8417238593101501, "metrics/particle/class_3/pid_accuracy": 0.9896682500839233, "metrics/particle/class_4/efficiency": 0.8706624507904053, "metrics/particle/class_4/pid_accuracy": 0.5760869383811951, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.772946834564209, "metrics/particle/count/bias_mean": 0.46000000834465027, "metrics/particle/count/mae": 2.868000030517578, "metrics/particle/count/prediction_mean": 96.18800354003906, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.027254516258835793, "metrics/particle/event/energy_response_mean": 1.0079922676086426, "metrics/particle/event/met_abs_error": 4.256570339202881, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028600642457604408, "metrics/particle/event/scalar_pt_response_mean": 1.0105406045913696, "metrics/particle/event/vector_pt_closure": 0.028514711186289787, "metrics/particle/matched/delta_eta_abs_mean": 0.008291841484606266, "metrics/particle/matched/delta_phi_abs_mean": 0.009598718956112862, "metrics/particle/matched/delta_r_mean": 0.01418604701757431, "metrics/particle/matched/energy_relative_abs_mean": 0.0864076316356659, "metrics/particle/matched/pid_accuracy": 0.973625123500824, "metrics/particle/matched/pt_relative_abs_mean": 0.08120711892843246, "metrics/particle/matching/duplicate_fraction": 0.026281865313649178, "metrics/particle/matching/efficiency": 0.8745194673538208, "metrics/particle/matching/f1": 0.8724233508110046, "metrics/particle/matching/purity": 0.8703372478485107, "metrics/particle/matching/target_coverage_dr0p05": 0.8573458194732666, "metrics/particle/matching/target_coverage_dr0p10": 0.9052732586860657, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7687615156173706, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9022229909896851}, "step_train_time": 0.8857371807098389, "step_valid_time": 2.01186766999308, "step_total_time": 414.0140960216522, "elapsed_training_loop_seconds": 414.0140960216522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054565247305411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0584824605024683, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9551130610867364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04391891891891892, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527983816587997, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9547297297297297, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9537630779615255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9541469993256911, "step_test_time": 3.4110290959943086, "step_plot_time": 3.2423621370107867, "step_validation_cycle_time": 9.132656674002646} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..a59712b8801ea82852d7448c493eab2c8deefdf5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 1.9404525756835938, "Classification": 0.01283281296491623, "Regression_pt": 0.0893421471118927, "Regression_eta": 0.004546793643385172, "Regression_sin_phi": 0.004053943790495396, "Regression_cos_phi": 0.003074423409998417, "Regression_energy": 0.08736851066350937, "Total": 11.417835235595703, "Total_unweighted": 2.141671207267791}, "train_group": {}, "valid": {"Classification_binary": 1.957563877105713, "Classification": 0.015690308064222336, "Regression_pt": 0.0933351069688797, "Regression_eta": 0.005789868533611298, "Regression_sin_phi": 0.00521493935957551, "Regression_cos_phi": 0.005444322247058153, "Regression_energy": 0.09277202188968658, "Total": 13.7440767288208, "Total_unweighted": 2.1758104441687465, "metrics/particle/class_1/efficiency": 0.9551327228546143, "metrics/particle/class_1/pid_accuracy": 0.985360860824585, "metrics/particle/class_2/efficiency": 0.5489192008972168, "metrics/particle/class_2/pid_accuracy": 0.8321243524551392, "metrics/particle/class_3/efficiency": 0.8445260524749756, "metrics/particle/class_3/pid_accuracy": 0.9878718256950378, "metrics/particle/class_4/efficiency": 0.8643532991409302, "metrics/particle/class_4/pid_accuracy": 0.6058394312858582, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.748792290687561, "metrics/particle/count/bias_mean": 0.5799999833106995, "metrics/particle/count/mae": 2.8919999599456787, "metrics/particle/count/prediction_mean": 96.30799865722656, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.02704085037112236, "metrics/particle/event/energy_response_mean": 1.0039359331130981, "metrics/particle/event/met_abs_error": 4.298669815063477, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027033716440200806, "metrics/particle/event/scalar_pt_response_mean": 1.0056618452072144, "metrics/particle/event/vector_pt_closure": 0.02859591506421566, "metrics/particle/matched/delta_eta_abs_mean": 0.00849783793091774, "metrics/particle/matched/delta_phi_abs_mean": 0.009634469635784626, "metrics/particle/matched/delta_r_mean": 0.014347315765917301, "metrics/particle/matched/energy_relative_abs_mean": 0.08696681261062622, "metrics/particle/matched/pid_accuracy": 0.9720672965049744, "metrics/particle/matched/pt_relative_abs_mean": 0.08094200491905212, "metrics/particle/matching/duplicate_fraction": 0.026415251195430756, "metrics/particle/matching/efficiency": 0.8766087293624878, "metrics/particle/matching/f1": 0.8739611506462097, "metrics/particle/matching/purity": 0.8713294863700867, "metrics/particle/matching/target_coverage_dr0p05": 0.8588082790374756, "metrics/particle/matching/target_coverage_dr0p10": 0.906150758266449, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7696807384490967, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9034765362739563}, "step_train_time": 0.8269116878509521, "step_valid_time": 2.2803535429993644, "step_total_time": 496.3248202800751, "elapsed_training_loop_seconds": 496.3248202800751, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0034381078891705, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06047382641764831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9574611748818366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568151147098516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042538825118163405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9561107359891965, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554655870445344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561699258260283, "step_test_time": 3.274050731997704, "step_plot_time": 2.834169469992048, "step_validation_cycle_time": 8.83237107199966} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..a2281723d2cddcc7eaefdb90c4a888693ccf2efe --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 1.9474401473999023, "Classification": 0.015419808216392994, "Regression_pt": 0.09663771092891693, "Regression_eta": 0.004627802409231663, "Regression_sin_phi": 0.005444207228720188, "Regression_cos_phi": 0.007597413379698992, "Regression_energy": 0.0933549776673317, "Total": 14.247711181640625, "Total_unweighted": 2.170522067230195}, "train_group": {}, "valid": {"Classification_binary": 1.9614049196243286, "Classification": 0.01585528254508972, "Regression_pt": 0.09359899908304214, "Regression_eta": 0.00580532057210803, "Regression_sin_phi": 0.005231945775449276, "Regression_cos_phi": 0.00544368289411068, "Regression_energy": 0.09310898184776306, "Total": 13.793187141418457, "Total_unweighted": 2.1804491323418915, "metrics/particle/class_1/efficiency": 0.9541593194007874, "metrics/particle/class_1/pid_accuracy": 0.9854387044906616, "metrics/particle/class_2/efficiency": 0.5477815866470337, "metrics/particle/class_2/pid_accuracy": 0.8390446305274963, "metrics/particle/class_3/efficiency": 0.8422069549560547, "metrics/particle/class_3/pid_accuracy": 0.9891005158424377, "metrics/particle/class_4/efficiency": 0.8706624507904053, "metrics/particle/class_4/pid_accuracy": 0.6014492511749268, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.75, "metrics/particle/count/bias_mean": 0.37599998712539673, "metrics/particle/count/mae": 2.864000082015991, "metrics/particle/count/prediction_mean": 96.10399627685547, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0262154508382082, "metrics/particle/event/energy_response_mean": 1.0017366409301758, "metrics/particle/event/met_abs_error": 4.1530914306640625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025845536962151527, "metrics/particle/event/scalar_pt_response_mean": 1.0037050247192383, "metrics/particle/event/vector_pt_closure": 0.02691543474793434, "metrics/particle/matched/delta_eta_abs_mean": 0.008392130956053734, "metrics/particle/matched/delta_phi_abs_mean": 0.009739913046360016, "metrics/particle/matched/delta_r_mean": 0.014372708275914192, "metrics/particle/matched/energy_relative_abs_mean": 0.0865228995680809, "metrics/particle/matched/pid_accuracy": 0.9728336334228516, "metrics/particle/matched/pt_relative_abs_mean": 0.08108692616224289, "metrics/particle/matching/duplicate_fraction": 0.025638891384005547, "metrics/particle/matching/efficiency": 0.8751880526542664, "metrics/particle/matching/f1": 0.8734726309776306, "metrics/particle/matching/purity": 0.8717639446258545, "metrics/particle/matching/target_coverage_dr0p05": 0.8568861484527588, "metrics/particle/matching/target_coverage_dr0p10": 0.9048972129821777, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7693046927452087, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9025572538375854}, "step_train_time": 0.8598263263702393, "step_valid_time": 2.110404037986882, "step_total_time": 583.0474646091461, "elapsed_training_loop_seconds": 583.0474646091461, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019361171999357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06264362513814636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9575185434929198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9594594594594594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9584880188997638, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04054054054054054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561699258260283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9581081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571380357745528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9575185434929198, "step_test_time": 3.2639620609988924, "step_plot_time": 2.7827852370101027, "step_validation_cycle_time": 8.521112939997693} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..348283a52252abd5afacb8378a76d49e2a4d12c8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 2.010732650756836, "Classification": 0.013527557253837585, "Regression_pt": 0.08220686763525009, "Regression_eta": 0.002716011367738247, "Regression_sin_phi": 0.004546624608337879, "Regression_cos_phi": 0.002532694721594453, "Regression_energy": 0.08435370773077011, "Total": 10.709996223449707, "Total_unweighted": 2.2006161140743643}, "train_group": {}, "valid": {"Classification_binary": 1.9650323390960693, "Classification": 0.015915602445602417, "Regression_pt": 0.09431719779968262, "Regression_eta": 0.005858524236828089, "Regression_sin_phi": 0.005228482652455568, "Regression_cos_phi": 0.005527683533728123, "Regression_energy": 0.09390036761760712, "Total": 13.883749961853027, "Total_unweighted": 2.1857801973819733, "metrics/particle/class_1/efficiency": 0.9549557566642761, "metrics/particle/class_1/pid_accuracy": 0.9866555333137512, "metrics/particle/class_2/efficiency": 0.5420932769775391, "metrics/particle/class_2/pid_accuracy": 0.8331584334373474, "metrics/particle/class_3/efficiency": 0.8401777744293213, "metrics/particle/class_3/pid_accuracy": 0.9907993078231812, "metrics/particle/class_4/efficiency": 0.8706624507904053, "metrics/particle/class_4/pid_accuracy": 0.6123188138008118, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7584540843963623, "metrics/particle/count/bias_mean": 0.25200000405311584, "metrics/particle/count/mae": 2.9560000896453857, "metrics/particle/count/prediction_mean": 95.9800033569336, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0271622184664011, "metrics/particle/event/energy_response_mean": 1.0023648738861084, "metrics/particle/event/met_abs_error": 4.227458953857422, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02669696882367134, "metrics/particle/event/scalar_pt_response_mean": 1.003906488418579, "metrics/particle/event/vector_pt_closure": 0.027643488720059395, "metrics/particle/matched/delta_eta_abs_mean": 0.008677409961819649, "metrics/particle/matched/delta_phi_abs_mean": 0.009645526297390461, "metrics/particle/matched/delta_r_mean": 0.014514359645545483, "metrics/particle/matched/energy_relative_abs_mean": 0.0865752100944519, "metrics/particle/matched/pid_accuracy": 0.9741898775100708, "metrics/particle/matched/pt_relative_abs_mean": 0.0808357447385788, "metrics/particle/matching/duplicate_fraction": 0.025213586166501045, "metrics/particle/matching/efficiency": 0.874226987361908, "metrics/particle/matching/f1": 0.8730778098106384, "metrics/particle/matching/purity": 0.8719316720962524, "metrics/particle/matching/target_coverage_dr0p05": 0.8535851836204529, "metrics/particle/matching/target_coverage_dr0p10": 0.9034765362739563, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7688450813293457, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9024319052696228}, "step_train_time": 0.8994288444519043, "step_valid_time": 2.315085624009953, "step_total_time": 669.3212752342224, "elapsed_training_loop_seconds": 669.3212752342224, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029768754522492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06135865242012806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9575185434929198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9581646423751687, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9578414839797639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04183535762483131, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956844234659474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574898785425101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9571669477234401, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9575185434929198, "step_test_time": 3.3521714300004533, "step_plot_time": 2.8768286909908056, "step_validation_cycle_time": 8.969370881008217} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..9738495dcc0a2e53ecbe7a4083c237569bed0ee7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 2.1339111328125, "Classification": 0.010758946649730206, "Regression_pt": 0.08541948348283768, "Regression_eta": 0.004202752839773893, "Regression_sin_phi": 0.004961615428328514, "Regression_cos_phi": 0.004692172631621361, "Regression_energy": 0.08323656022548676, "Total": 11.988337516784668, "Total_unweighted": 2.3271826640702784}, "train_group": {}, "valid": {"Classification_binary": 1.967637062072754, "Classification": 0.01580207236111164, "Regression_pt": 0.09452053159475327, "Regression_eta": 0.0058157858438789845, "Regression_sin_phi": 0.005282300524413586, "Regression_cos_phi": 0.005558227654546499, "Regression_energy": 0.09407369792461395, "Total": 13.895018577575684, "Total_unweighted": 2.188689677976072, "metrics/particle/class_1/efficiency": 0.9548672437667847, "metrics/particle/class_1/pid_accuracy": 0.9857275485992432, "metrics/particle/class_2/efficiency": 0.5403867959976196, "metrics/particle/class_2/pid_accuracy": 0.8336842060089111, "metrics/particle/class_3/efficiency": 0.8444294333457947, "metrics/particle/class_3/pid_accuracy": 0.9903879165649414, "metrics/particle/class_4/efficiency": 0.8580441474914551, "metrics/particle/class_4/pid_accuracy": 0.591911792755127, "metrics/particle/class_5/efficiency": 0.995192289352417, "metrics/particle/class_5/pid_accuracy": 0.7777777910232544, "metrics/particle/count/bias_mean": 0.5040000081062317, "metrics/particle/count/mae": 2.8559999465942383, "metrics/particle/count/prediction_mean": 96.23200225830078, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.027722952887415886, "metrics/particle/event/energy_response_mean": 1.0051534175872803, "metrics/particle/event/met_abs_error": 4.351737022399902, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027975451201200485, "metrics/particle/event/scalar_pt_response_mean": 1.006001353263855, "metrics/particle/event/vector_pt_closure": 0.028141222894191742, "metrics/particle/matched/delta_eta_abs_mean": 0.008292784914374352, "metrics/particle/matched/delta_phi_abs_mean": 0.009678804315626621, "metrics/particle/matched/delta_r_mean": 0.014251037500798702, "metrics/particle/matched/energy_relative_abs_mean": 0.08713649213314056, "metrics/particle/matched/pid_accuracy": 0.9736139178276062, "metrics/particle/matched/pt_relative_abs_mean": 0.0812305212020874, "metrics/particle/matching/duplicate_fraction": 0.026103582233190536, "metrics/particle/matching/efficiency": 0.8757312297821045, "metrics/particle/matching/f1": 0.8734319806098938, "metrics/particle/matching/purity": 0.871144711971283, "metrics/particle/matching/target_coverage_dr0p05": 0.8587247133255005, "metrics/particle/matching/target_coverage_dr0p10": 0.9056493639945984, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7691793441772461, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9023901224136353}, "step_train_time": 0.8806960582733154, "step_valid_time": 2.024771005002549, "step_total_time": 755.8246786594391, "elapsed_training_loop_seconds": 755.8246786594391, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003635859311257, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0624517224998431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548213081591369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9567567567567568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9557880526493419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043243243243243246, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9541469993256911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956081081081081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551130610867364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548213081591369, "step_test_time": 3.2478579629969317, "step_plot_time": 2.788092049988336, "step_validation_cycle_time": 8.425506612009485} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..4a2e0c1eab523b57d67e15c7d1907258c02364fb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..77827988f9aa8715fd4689eafdeefd22616984ce --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4588e7b1723abb88e2cb8cd78061dcb87573e7e74215ef5817e986e17fb17ea3 +size 4498 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b5e3facc2850bba26791a2aa6d9eeb2a7765a0c7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..7215406a7a9fd2b1c8250061352697faefa31fea Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1ff85a0eb919257c1397e18b22db2f2c98edd2ce Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..fd5347e11da93caa1954d9f184ca3f32ce693364 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..326e8ae466013718e24e66aad427c345ca6faf0b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..08983257dddb8890e31df4c83d5404fc3c9dab68 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3acfec215e7e232844aa709a1f5b37242ec41d69 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..d291b158098560385682d66daabf1d573a0f631a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d1687d6f7f8c6dab433b02741b6d1509ccc1a0b7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ed188904ae9214419eb41b43b4700f7585fb721d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a5d60aa3bba4a930aa21babdba933f5050415e18 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..a3cb3462d5d90aed046b212c715f7626f7d8231b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/plots_step_33000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/train/events.out.tfevents.1794732881.workergpu212.2312997.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/train/events.out.tfevents.1794732881.workergpu212.2312997.0 new file mode 100644 index 0000000000000000000000000000000000000000..af936312a070aad5cd84b369e8777b1e26111131 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/train/events.out.tfevents.1794732881.workergpu212.2312997.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6870f4ed409649e3aba907a2b7043bda3778a5191db247ed56c5a5dca195ba03 +size 1214204 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/valid/events.out.tfevents.1794732881.workergpu212.2312997.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/valid/events.out.tfevents.1794732881.workergpu212.2312997.1 new file mode 100644 index 0000000000000000000000000000000000000000..916227a6d5e0b02352ea0ba799db21939f2485c4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/runs/valid/events.out.tfevents.1794732881.workergpu212.2312997.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e30793d4c2362a463495f4bfaa465797a1277557753ddee3c49894fd83f13b3 +size 16910394 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c754072de9fe9dca58f04ad1f4b7c86f35abf980 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_20mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68a982035ef2c2816a7ef244ab10f8bf5bce0386 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a11a70aec80ace826a2dd154bb0485d8f7a6014b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log @@ -0,0 +1,2630 @@ +rank0 [2026-11-15 03:54:40,421] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-15 03:54:40,422] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 03:54:40,670] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:54:40,670] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:54:40,670] INFO : Instantiating model +rank0 [2026-11-15 03:54:40,689] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:54:40,689] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:54:40,690] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:54:40,690] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:54:40,690] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:54:40,690] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:54:40,690] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,693] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:54:40,693] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,695] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:54:40,695] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,697] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:54:40,697] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,699] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:54:40,699] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,702] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:54:40,702] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,704] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 03:54:40,704] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:54:40,704] INFO : Initializing output DNNs +rank0 [2026-11-15 03:54:40,705] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,708] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,714] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:54:40,714] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:54:40,714] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:54:40,714] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:54:40,715] INFO : task_queries enabled=True +rank0 [2026-11-15 03:54:40,715] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:54:40,715] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:54:40,715] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:54:40,715] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:54:40,715] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:54:40,715] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:54:40,715] INFO : output_mode=elementwise +rank0 [2026-11-15 03:54:40,715] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:54:40,715] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:54:40,715] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:54:40,715] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:54:40,716] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:54:40,716] INFO : MLPF __init__ done +rank0 [2026-11-15 03:54:40,716] INFO : Instantiated model +rank0 [2026-11-15 03:54:40,716] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:54:40,724] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:54:40,725] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:54:40,794] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:54:40,794] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:54:40,795] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:54:40,853] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:54:40,853] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:54:40,854] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:54:40,854] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:54:40,854] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:54:40,855] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:54:40,857] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643 +rank0 [2026-11-15 03:54:40,857] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643 +rank0 [2026-11-15 03:54:40,857] INFO : Getting dataloaders +rank0 [2026-11-15 03:54:40,857] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:41,048] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,127] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,176] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,226] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,269] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,311] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,352] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,443] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,479] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,527] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,527] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:54:41,528] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:54:41,528] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:54:41,528] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:41,540] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,553] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,564] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,577] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,589] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,601] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,614] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,626] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,638] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,650] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,650] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:54:41,650] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:54:41,679] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:41,834] INFO : Got dataloaders +rank0 [2026-11-15 03:54:41,835] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:54:41,835] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:54:41,835] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:54:41,839] INFO : Creating train_iterator +rank0 [2026-11-15 03:54:41,839] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:54:41,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:42,028] INFO : Created train_iterator +rank0 [2026-11-15 03:54:50,567] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9681692945957183, 'Classification': 0.014965717205777765, 'Regression_pt': 0.09860456176102161, 'Regression_eta': 0.005572905512526632, 'Regression_sin_phi': 0.005349073330871761, 'Regression_cos_phi': 0.005312690008431673, 'Regression_energy': 0.0971477759256959}, weights={'Classification_binary': 1.0, 'Classification': 131.51185607910156, 'Regression_pt': 19.96022605895996, 'Regression_eta': 353.16754150390625, 'Regression_sin_phi': 367.9458312988281, 'Regression_cos_phi': 370.4656677246094, 'Regression_energy': 20.259540557861328} +rank0 [2026-11-15 03:55:24,954] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:55:24,954] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:58,430] INFO : Step 1000/500000 rank0 | Train Loss: 12.4813 | Train Loss Unweighted: 2.3390 | Train Loss Components: Classification: 0.0144 | Classification_binary: 2.1458 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0820 | Regression_eta: 0.0043 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8913 | Classification_binary: 2.1458 | Regression_cos_phi: 1.8440 | Regression_energy: 1.6614 | Regression_eta: 1.5243 | Regression_pt: 1.6512 | Regression_sin_phi: 1.7633 | LR: 2.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8341s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0022 | max_backward: 0.0275 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.3369 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2662.7 | uss_MB: 1449.7 | child_rss_MB: 23085.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:55:58,431] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:58,431] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:58,431] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:58,558] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:55:58,675] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:58,732] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:58,772] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:58,810] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:58,844] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:58,883] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:58,923] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:58,958] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:59,013] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:59,069] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:59,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:59,155] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:59,189] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:59,229] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:59,268] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:59,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:59,308] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:59,309] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:59,311] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:59,313] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:59,314] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:59,316] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:59,318] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:59,319] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:59,321] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:59,322] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:59,324] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:59,325] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:59,327] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:59,329] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:59,330] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:56:00,331] INFO : VALIDATION | Step=1000/500000 | Train Loss=12.4813 | Valid Loss=13.7349 | Train Loss Unweighted=2.3390 | Valid Loss Unweighted=2.1596 | Stale=0 | ETA=652.8m +rank0 [2026-11-15 03:56:00,331] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:56:00,480] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:56:00,488] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:00,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:56:01,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:56:01,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:56:01,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:56:01,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:56:01,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:56:01,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:56:01,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:56:02,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:56:02,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:56:02,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:56:02,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:56:02,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:56:02,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:56:03,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:56:03,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:56:03,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:56:03,519] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:56:03,537] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:03,929] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:04,253] INFO : Plotted number of elements +rank0 [2026-11-15 03:56:05,222] INFO : Plotted jets +rank0 [2026-11-15 03:56:06,454] INFO : Plotted jet ratio +rank0 [2026-11-15 03:56:16,510] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:56:16,510] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:00,189] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:57:00,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:24,089] INFO : Step 2000/500000 rank0 | Train Loss: 15.3531 | Train Loss Unweighted: 2.1627 | Train Loss Components: Classification: 0.0138 | Classification_binary: 1.9078 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1130 | Regression_eta: 0.0066 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8188 | Classification_binary: 1.9078 | Regression_cos_phi: 2.7862 | Regression_energy: 2.2891 | Regression_eta: 2.3318 | Regression_pt: 2.1625 | Regression_sin_phi: 2.0567 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8859s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.3387 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2818.1 | uss_MB: 1620.3 | child_rss_MB: 23091.4 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:57:24,091] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:57:24,153] INFO : Running validation on rank0 +rank0 [2026-11-15 03:57:24,153] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:57:24,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:57:24,271] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:57:24,357] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:57:24,435] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:57:24,485] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:57:24,531] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:57:24,577] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:57:24,623] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:57:24,669] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:57:24,738] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:57:24,814] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:57:24,892] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:57:24,939] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:57:24,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:57:25,031] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:57:25,078] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:57:25,124] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:57:25,196] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:57:25,201] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:57:25,203] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:57:25,204] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:57:25,206] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:57:25,208] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:57:25,209] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:57:25,211] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:57:25,212] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:57:25,214] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:57:25,215] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:57:25,217] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:57:25,219] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:57:25,220] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:57:25,222] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:57:25,223] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:57:26,209] INFO : VALIDATION | Step=2000/500000 | Train Loss=15.3531 | Valid Loss=13.7213 | Train Loss Unweighted=2.1627 | Valid Loss Unweighted=2.1631 | Stale=0 | ETA=682.1m +rank0 [2026-11-15 03:57:26,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:57:26,363] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:57:26,378] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:26,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:57:26,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:57:27,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:57:27,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:57:27,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:57:27,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:57:27,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:57:27,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:57:28,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:57:28,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:57:28,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:57:28,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:57:28,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:57:28,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:57:28,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:57:29,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:57:29,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:57:29,391] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:57:29,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:29,781] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:30,025] INFO : Plotted number of elements +rank0 [2026-11-15 03:57:30,946] INFO : Plotted jets +rank0 [2026-11-15 03:57:32,132] INFO : Plotted jet ratio +rank0 [2026-11-15 03:57:50,559] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:57:50,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:31,537] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:58:31,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:46,265] INFO : Step 3000/500000 rank0 | Train Loss: 10.8768 | Train Loss Unweighted: 2.0285 | Train Loss Components: Classification: 0.0124 | Classification_binary: 1.8641 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0701 | Regression_eta: 0.0035 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6351 | Classification_binary: 1.8641 | Regression_cos_phi: 1.7712 | Regression_energy: 1.4211 | Regression_eta: 1.2206 | Regression_pt: 1.3831 | Regression_sin_phi: 1.5817 | LR: 2.05e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8762s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0351 | max_loss: 0.0029 | max_backward: 0.0272 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.3022 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2874.8 | uss_MB: 1677.0 | child_rss_MB: 23089.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:58:46,266] INFO : Running validation on rank0 +rank0 [2026-11-15 03:58:46,266] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:58:46,267] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:58:46,388] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:58:46,485] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:58:46,577] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:58:46,634] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:58:46,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:58:46,739] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:58:46,790] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:58:46,841] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:58:46,915] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:58:47,000] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:58:47,086] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:58:47,137] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:58:47,186] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:58:47,234] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:58:47,282] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:58:47,333] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:58:47,416] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:58:47,421] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:58:47,423] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:58:47,425] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:58:47,427] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:58:47,428] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:58:47,430] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:58:47,432] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:58:47,433] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:58:47,435] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:58:47,436] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:58:47,438] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:58:47,440] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:58:47,441] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:58:47,443] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:58:47,444] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:58:48,667] INFO : VALIDATION | Step=3000/500000 | Train Loss=10.8768 | Valid Loss=13.7010 | Train Loss Unweighted=2.0285 | Valid Loss Unweighted=2.1665 | Stale=0 | ETA=681.5m +rank0 [2026-11-15 03:58:48,667] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:58:48,812] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:58:48,821] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:48,887] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:58:49,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:58:49,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:58:49,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:58:49,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:58:50,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:58:50,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:58:50,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:58:50,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:58:50,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:58:50,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:58:51,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:58:51,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:58:51,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:58:51,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:58:51,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:58:51,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:58:51,907] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:58:51,914] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:52,290] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:52,537] INFO : Plotted number of elements +rank0 [2026-11-15 03:58:53,996] INFO : Plotted jets +rank0 [2026-11-15 03:58:54,737] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:23,612] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:59:23,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:04,627] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:00:04,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:10,553] INFO : Step 4000/500000 rank0 | Train Loss: 16.3001 | Train Loss Unweighted: 2.4321 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.1677 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1137 | Regression_eta: 0.0042 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.3967 | Classification_binary: 2.1677 | Regression_cos_phi: 3.6925 | Regression_energy: 2.3030 | Regression_eta: 1.4852 | Regression_pt: 2.2545 | Regression_sin_phi: 2.0005 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0252 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2904.3 | uss_MB: 1706.5 | child_rss_MB: 23091.3 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:00:10,555] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:00:10,605] INFO : Running validation on rank0 +rank0 [2026-11-15 04:00:10,605] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:00:10,605] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:00:10,713] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:00:10,799] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:00:10,883] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:00:10,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:00:10,983] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:00:11,030] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:00:11,077] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:00:11,125] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:00:11,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:00:11,278] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:00:11,359] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:00:11,407] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:00:11,453] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:00:11,498] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:00:11,543] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:00:11,590] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:00:11,664] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:00:11,669] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:00:11,671] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:00:11,673] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:00:11,674] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:00:11,676] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:00:11,677] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:00:11,679] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:00:11,680] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:00:11,682] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:00:11,684] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:00:11,685] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:00:11,687] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:00:11,689] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:00:11,690] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:00:11,692] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:12,632] INFO : VALIDATION | Step=4000/500000 | Train Loss=16.3001 | Valid Loss=13.7493 | Train Loss Unweighted=2.4321 | Valid Loss Unweighted=2.1664 | Stale=1 | ETA=683.6m +rank0 [2026-11-15 04:00:12,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:00:12,779] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:00:12,787] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:12,849] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:13,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:13,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:13,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:13,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:14,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:14,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:14,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:14,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:14,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:14,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:15,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:15,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:15,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:15,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:15,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:15,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:15,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:00:16,005] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:16,392] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:16,641] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:18,243] INFO : Plotted jets +rank0 [2026-11-15 04:00:18,989] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:56,105] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:00:56,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:01:33,838] INFO : Step 5000/500000 rank0 | Train Loss: 14.1188 | Train Loss Unweighted: 2.1746 | Train Loss Components: Classification: 0.0190 | Classification_binary: 1.9344 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1028 | Regression_eta: 0.0074 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.5041 | Classification_binary: 1.9344 | Regression_cos_phi: 1.3276 | Regression_energy: 2.0820 | Regression_eta: 2.6307 | Regression_pt: 2.0585 | Regression_sin_phi: 1.5815 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0266 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.3572 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2931.1 | uss_MB: 1733.4 | child_rss_MB: 23092.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:01:33,840] INFO : Running validation on rank0 +rank0 [2026-11-15 04:01:33,840] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:01:33,840] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:01:33,945] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:01:34,027] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:01:34,105] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:01:34,156] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:01:34,204] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:01:34,252] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:01:34,299] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:01:34,346] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:01:34,416] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:01:34,494] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:01:34,572] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:01:34,620] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:01:34,666] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:01:34,712] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:01:34,759] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:01:34,806] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:01:34,880] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:01:34,885] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:01:34,887] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:01:34,888] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:01:34,890] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:01:34,892] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:01:34,893] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:01:34,895] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:01:34,896] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:01:34,898] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:01:34,900] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:01:34,901] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:01:34,903] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:01:34,904] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:01:34,906] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:01:34,908] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:01:35,856] INFO : VALIDATION | Step=5000/500000 | Train Loss=14.1188 | Valid Loss=13.7680 | Train Loss Unweighted=2.1746 | Valid Loss Unweighted=2.1706 | Stale=2 | ETA=683.1m +rank0 [2026-11-15 04:01:35,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:01:36,023] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:01:36,031] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:36,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:01:36,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:01:36,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:01:36,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:01:37,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:01:37,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:01:37,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:01:37,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:01:37,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:01:38,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:01:38,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:01:38,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:01:38,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:01:38,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:01:38,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:01:39,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:01:39,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:01:39,265] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:01:39,272] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:39,688] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:39,981] INFO : Plotted number of elements +rank0 [2026-11-15 04:01:41,028] INFO : Plotted jets +rank0 [2026-11-15 04:01:42,502] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:47,286] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:01:47,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:27,647] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:02:27,647] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:55,829] INFO : Step 6000/500000 rank0 | Train Loss: 11.4178 | Train Loss Unweighted: 2.1417 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.9405 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0874 | Regression_eta: 0.0045 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6877 | Classification_binary: 1.9405 | Regression_cos_phi: 1.1390 | Regression_energy: 1.7700 | Regression_eta: 1.6058 | Regression_pt: 1.7833 | Regression_sin_phi: 1.4916 | LR: 2.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8260s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0876 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2869.3 | uss_MB: 1671.6 | child_rss_MB: 23095.7 | allocated_MB: 170.9 | reserved_MB: 694.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:02:55,831] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:02:55,881] INFO : Running validation on rank0 +rank0 [2026-11-15 04:02:55,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:02:55,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:02:56,015] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:02:56,107] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:02:56,190] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:02:56,242] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:02:56,291] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:02:56,340] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:02:56,391] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:02:56,442] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:02:56,517] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:02:56,599] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:02:56,691] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:02:56,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:02:56,799] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:02:56,856] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:02:56,916] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:02:56,972] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:02:57,059] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:02:57,064] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:02:57,066] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:02:57,068] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:02:57,070] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:02:57,072] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:02:57,074] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:02:57,077] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:02:57,077] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:02:57,079] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:02:57,081] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:02:57,083] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:02:57,085] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:02:57,087] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:02:57,089] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:02:57,091] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:02:58,167] INFO : VALIDATION | Step=6000/500000 | Train Loss=11.4178 | Valid Loss=13.7441 | Train Loss Unweighted=2.1417 | Valid Loss Unweighted=2.1758 | Stale=3 | ETA=681.1m +rank0 [2026-11-15 04:02:58,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:02:58,305] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:02:58,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:58,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:02:58,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:02:59,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:02:59,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:02:59,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:02:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:02:59,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:02:59,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:03:00,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:03:00,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:03:00,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:03:00,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:03:00,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:03:00,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:03:01,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:03:01,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:03:01,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:03:01,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:03:01,448] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:03:01,825] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:03:02,073] INFO : Plotted number of elements +rank0 [2026-11-15 04:03:03,536] INFO : Plotted jets +rank0 [2026-11-15 04:03:04,267] INFO : Plotted jet ratio +rank0 [2026-11-15 04:03:18,002] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:03:18,003] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:01,973] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:04:01,973] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:22,773] INFO : Step 7000/500000 rank0 | Train Loss: 14.2477 | Train Loss Unweighted: 2.1705 | Train Loss Components: Classification: 0.0154 | Classification_binary: 1.9474 | Regression_cos_phi: 0.0076 | Regression_energy: 0.0934 | Regression_eta: 0.0046 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.0279 | Classification_binary: 1.9474 | Regression_cos_phi: 2.8146 | Regression_energy: 1.8913 | Regression_eta: 1.6344 | Regression_pt: 1.9289 | Regression_sin_phi: 2.0032 | LR: 2.26e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0300 | max_loss: 0.0027 | max_backward: 0.0276 | max_optimizer_step: 0.0894 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2980.5 | uss_MB: 1782.8 | child_rss_MB: 23093.7 | allocated_MB: 170.4 | reserved_MB: 696.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:04:22,774] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:22,774] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:22,774] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:22,883] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:04:22,967] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:23,047] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:23,096] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:23,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:23,193] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:23,245] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:23,297] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:23,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:23,457] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:23,538] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:23,587] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:23,636] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:23,685] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:23,738] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:23,791] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:23,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:23,876] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:23,878] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:23,880] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:23,882] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:23,883] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:23,885] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:23,886] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:23,888] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:23,889] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:23,891] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:23,893] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:23,894] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:23,896] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:23,897] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:23,899] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:24,889] INFO : VALIDATION | Step=7000/500000 | Train Loss=14.2477 | Valid Loss=13.7932 | Train Loss Unweighted=2.1705 | Valid Loss Unweighted=2.1804 | Stale=4 | ETA=684.4m +rank0 [2026-11-15 04:04:24,889] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:04:25,027] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:04:25,035] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:25,091] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:25,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:25,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:25,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:26,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:26,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:26,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:26,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:26,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:26,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:27,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:27,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:27,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:27,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:27,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:27,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:28,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:28,151] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:04:28,159] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:28,543] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:28,789] INFO : Plotted number of elements +rank0 [2026-11-15 04:04:30,194] INFO : Plotted jets +rank0 [2026-11-15 04:04:30,929] INFO : Plotted jet ratio +rank0 [2026-11-15 04:04:54,328] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:04:54,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:37,912] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:05:37,912] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:48,785] INFO : Step 8000/500000 rank0 | Train Loss: 10.7100 | Train Loss Unweighted: 2.2006 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.0107 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0844 | Regression_eta: 0.0027 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7790 | Classification_binary: 2.0107 | Regression_cos_phi: 0.9383 | Regression_energy: 1.7090 | Regression_eta: 0.9592 | Regression_pt: 1.6409 | Regression_sin_phi: 1.6729 | LR: 2.34e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.8980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0314 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0882 | max_tensorboard_logging: 0.3182 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3000.4 | uss_MB: 1802.7 | child_rss_MB: 23094.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 25.6 +rank0 [2026-11-15 04:05:48,787] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:05:48,843] INFO : Running validation on rank0 +rank0 [2026-11-15 04:05:48,843] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:05:48,843] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:05:48,972] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:05:49,063] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:05:49,148] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:05:49,206] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:05:49,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:05:49,316] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:05:49,374] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:05:49,428] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:05:49,502] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:05:49,584] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:05:49,669] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:05:49,727] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:05:49,782] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:05:49,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:05:49,894] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:05:49,946] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:05:50,023] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:05:50,028] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:05:50,030] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:05:50,032] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:05:50,034] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:05:50,035] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:05:50,037] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:05:50,039] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:05:50,040] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:05:50,042] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:05:50,044] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:05:50,045] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:05:50,047] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:05:50,049] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:05:50,051] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:05:50,052] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:05:51,162] INFO : VALIDATION | Step=8000/500000 | Train Loss=10.7100 | Valid Loss=13.8837 | Train Loss Unweighted=2.2006 | Valid Loss Unweighted=2.1858 | Stale=5 | ETA=686.1m +rank0 [2026-11-15 04:05:51,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:05:51,319] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:05:51,327] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:51,384] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:05:51,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:05:52,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:05:52,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:05:52,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:05:52,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:05:52,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:05:52,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:05:53,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:05:53,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:05:53,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:05:53,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:05:53,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:05:53,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:54,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:54,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:54,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:54,512] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:05:54,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:54,933] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:55,187] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:56,639] INFO : Plotted jets +rank0 [2026-11-15 04:05:57,383] INFO : Plotted jet ratio +rank0 [2026-11-15 04:06:31,090] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:06:31,090] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:14,204] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:07:14,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:15,635] INFO : Step 9000/500000 rank0 | Train Loss: 11.9883 | Train Loss Unweighted: 2.3272 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.1339 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0832 | Regression_eta: 0.0042 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4149 | Classification_binary: 2.1339 | Regression_cos_phi: 1.7383 | Regression_energy: 1.6863 | Regression_eta: 1.4843 | Regression_pt: 1.7050 | Regression_sin_phi: 1.8256 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8803s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0278 | max_optimizer_step: 0.0885 | max_tensorboard_logging: 0.3440 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2942.8 | uss_MB: 1745.2 | child_rss_MB: 23095.6 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:07:15,637] INFO : Running validation on rank0 +rank0 [2026-11-15 04:07:15,637] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:07:15,637] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:07:15,756] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:07:15,841] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:07:15,921] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:07:15,971] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:07:16,018] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:07:16,064] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:07:16,111] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:07:16,158] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:07:16,229] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:07:16,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:07:16,393] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:07:16,442] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:07:16,489] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:07:16,536] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:07:16,583] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:07:16,630] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:07:16,703] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:07:16,708] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:07:16,710] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:07:16,712] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:07:16,713] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:07:16,715] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:07:16,716] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:07:16,718] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:07:16,719] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:07:16,721] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:07:16,722] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:07:16,724] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:07:16,726] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:07:16,727] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:07:16,729] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:07:16,730] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:07:17,666] INFO : VALIDATION | Step=9000/500000 | Train Loss=11.9883 | Valid Loss=13.8950 | Train Loss Unweighted=2.3272 | Valid Loss Unweighted=2.1887 | Stale=6 | ETA=687.2m +rank0 [2026-11-15 04:07:17,667] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:07:17,829] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:07:17,837] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:17,897] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:07:18,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:07:18,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:07:18,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:07:18,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:07:19,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:07:19,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:07:19,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:07:19,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:07:19,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:07:19,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:07:20,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:07:20,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:07:20,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:07:20,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:07:20,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:07:20,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:07:20,912] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:07:20,920] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:21,306] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:21,563] INFO : Plotted number of elements +rank0 [2026-11-15 04:07:22,969] INFO : Plotted jets +rank0 [2026-11-15 04:07:23,694] INFO : Plotted jet ratio +rank0 [2026-11-15 04:08:07,158] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:08:07,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:41,904] INFO : Step 10000/500000 rank0 | Train Loss: 10.8362 | Train Loss Unweighted: 2.0714 | Train Loss Components: Classification: 0.0112 | Classification_binary: 1.8592 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0968 | Regression_eta: 0.0032 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4718 | Classification_binary: 1.8592 | Regression_cos_phi: 1.2490 | Regression_energy: 1.9612 | Regression_eta: 1.1132 | Regression_pt: 1.8790 | Regression_sin_phi: 1.3028 | LR: 2.52e-05 | DataLoad Time: 0.0029s | Model Forward Time: 0.8338s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0022 | max_backward: 0.0274 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3052.6 | uss_MB: 1854.9 | child_rss_MB: 23096.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:08:41,906] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:08:41,955] INFO : Running validation on rank0 +rank0 [2026-11-15 04:08:41,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:08:41,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:08:42,078] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:08:42,177] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:08:42,264] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:08:42,323] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:08:42,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:08:42,431] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:08:42,484] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:08:42,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:08:42,611] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:08:42,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:08:42,771] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:08:42,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:08:42,876] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:08:42,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:08:42,980] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:08:43,029] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:08:43,105] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:08:43,110] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:08:43,112] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:08:43,113] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:08:43,115] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:08:43,117] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:08:43,118] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:08:43,120] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:08:43,121] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:08:43,123] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:08:43,124] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:08:43,126] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:08:43,127] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:08:43,129] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:08:43,131] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:08:43,132] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:08:44,160] INFO : VALIDATION | Step=10000/500000 | Train Loss=10.8362 | Valid Loss=13.9099 | Train Loss Unweighted=2.0714 | Valid Loss Unweighted=2.1926 | Stale=7 | ETA=687.9m +rank0 [2026-11-15 04:08:44,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:08:44,295] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:08:44,316] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:44,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:08:44,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:08:45,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:08:45,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:08:45,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:08:45,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:08:45,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:08:45,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:08:46,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:08:46,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:08:46,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:08:46,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:08:46,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:08:46,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:08:47,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:08:47,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:08:47,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:08:47,484] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:08:47,492] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:47,882] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:48,130] INFO : Plotted number of elements +rank0 [2026-11-15 04:08:49,076] INFO : Plotted jets +rank0 [2026-11-15 04:08:50,351] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:00,053] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:09:00,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:09:43,184] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:09:43,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:08,578] INFO : Step 11000/500000 rank0 | Train Loss: 9.0098 | Train Loss Unweighted: 2.0489 | Train Loss Components: Classification: 0.0101 | Classification_binary: 1.8939 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0668 | Regression_eta: 0.0027 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.3313 | Classification_binary: 1.8939 | Regression_cos_phi: 1.2002 | Regression_energy: 1.3539 | Regression_eta: 0.9632 | Regression_pt: 1.3916 | Regression_sin_phi: 0.8758 | LR: 2.63e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0251 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3075.7 | uss_MB: 1878.0 | child_rss_MB: 23094.5 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 04:10:08,580] INFO : Running validation on rank0 +rank0 [2026-11-15 04:10:08,580] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:10:08,580] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:10:08,697] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:10:08,781] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:10:08,861] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:10:08,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:10:08,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:10:09,009] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:10:09,058] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:10:09,106] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:10:09,177] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:10:09,255] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:10:09,334] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:10:09,381] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:10:09,427] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:10:09,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:10:09,518] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:10:09,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:10:09,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:10:09,642] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:10:09,644] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:10:09,645] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:10:09,647] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:10:09,648] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:10:09,651] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:10:09,652] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:10:09,653] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:10:09,655] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:10:09,656] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:10:09,658] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:10:09,659] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:10:09,661] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:10:09,662] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:10:09,667] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:10:10,647] INFO : VALIDATION | Step=11000/500000 | Train Loss=9.0098 | Valid Loss=14.0044 | Train Loss Unweighted=2.0489 | Valid Loss Unweighted=2.1949 | Stale=8 | ETA=688.2m +rank0 [2026-11-15 04:10:10,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:10:10,789] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:10:10,798] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:10,855] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:10:11,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:10:11,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:10:11,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:10:11,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:10:12,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:10:12,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:10:12,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:10:12,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:10:12,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:10:13,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:10:13,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:10:13,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:10:13,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:10:13,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:10:13,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:10:14,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:10:14,153] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:10:14,171] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:14,597] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:14,870] INFO : Plotted number of elements +rank0 [2026-11-15 04:10:15,883] INFO : Plotted jets +rank0 [2026-11-15 04:10:17,363] INFO : Plotted jet ratio +rank0 [2026-11-15 04:10:36,145] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:10:36,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:19,053] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:11:19,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:35,081] INFO : Step 12000/500000 rank0 | Train Loss: 7.5518 | Train Loss Unweighted: 1.6791 | Train Loss Components: Classification: 0.0088 | Classification_binary: 1.5423 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0606 | Regression_eta: 0.0022 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.1604 | Classification_binary: 1.5423 | Regression_cos_phi: 0.9616 | Regression_energy: 1.2272 | Regression_eta: 0.7621 | Regression_pt: 1.2131 | Regression_sin_phi: 0.6851 | LR: 2.75e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8389s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0832 | max_tensorboard_logging: 0.3018 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3098.7 | uss_MB: 1901.1 | child_rss_MB: 23097.4 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:11:35,083] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:11:35,132] INFO : Running validation on rank0 +rank0 [2026-11-15 04:11:35,132] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:11:35,132] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:11:35,236] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:11:35,319] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:11:35,400] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:11:35,450] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:11:35,496] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:11:35,542] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:11:35,589] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:11:35,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:11:35,707] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:11:35,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:11:35,866] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:11:35,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:11:35,961] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:11:36,007] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:11:36,053] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:11:36,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:11:36,173] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:11:36,178] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:11:36,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:11:36,182] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:11:36,183] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:11:36,185] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:11:36,187] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:11:36,188] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:11:36,190] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:11:36,191] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:11:36,193] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:11:36,195] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:11:36,196] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:11:36,198] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:11:36,200] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:11:36,201] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:11:37,147] INFO : VALIDATION | Step=12000/500000 | Train Loss=7.5518 | Valid Loss=13.9547 | Train Loss Unweighted=1.6791 | Valid Loss Unweighted=2.2042 | Stale=9 | ETA=688.2m +rank0 [2026-11-15 04:11:37,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:11:37,306] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:11:37,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:37,379] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:11:37,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:11:38,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:11:38,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:11:38,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:11:38,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:11:38,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:11:38,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:11:39,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:11:39,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:11:39,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:11:39,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:11:39,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:11:39,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:11:40,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:11:40,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:11:40,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:11:40,424] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:11:40,431] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:40,819] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:41,067] INFO : Plotted number of elements +rank0 [2026-11-15 04:11:42,560] INFO : Plotted jets +rank0 [2026-11-15 04:11:43,291] INFO : Plotted jet ratio +rank0 [2026-11-15 04:12:11,426] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:12:11,426] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:53,994] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:12:53,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:00,449] INFO : Step 13000/500000 rank0 | Train Loss: 11.5497 | Train Loss Unweighted: 2.2882 | Train Loss Components: Classification: 0.0112 | Classification_binary: 2.1230 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0704 | Regression_eta: 0.0031 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4790 | Classification_binary: 2.1230 | Regression_cos_phi: 1.7573 | Regression_energy: 1.4261 | Regression_eta: 1.1067 | Regression_pt: 1.3892 | Regression_sin_phi: 2.2686 | LR: 2.88e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.8832s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0276 | max_optimizer_step: 0.0842 | max_tensorboard_logging: 0.3429 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3129.5 | uss_MB: 1931.8 | child_rss_MB: 23096.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:13:00,450] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:00,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:00,451] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:00,560] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:13:00,641] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:00,721] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:00,769] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:00,815] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:00,861] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:00,908] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:00,955] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:01,025] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:01,105] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:01,183] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:01,230] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:01,275] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:01,321] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:01,367] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:01,413] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:01,487] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:01,492] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:01,493] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:01,495] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:01,496] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:01,498] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:01,500] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:01,501] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:01,503] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:01,504] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:01,506] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:01,507] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:01,509] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:01,510] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:01,512] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:01,514] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:02,496] INFO : VALIDATION | Step=13000/500000 | Train Loss=11.5497 | Valid Loss=14.0858 | Train Loss Unweighted=2.2882 | Valid Loss Unweighted=2.2122 | Stale=10 | ETA=687.2m +rank0 [2026-11-15 04:13:02,496] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:13:02,656] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:13:02,664] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:02,717] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:03,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:03,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:03,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:03,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:03,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:04,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:04,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:04,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:04,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:04,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:04,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:04,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:05,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:05,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:05,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:13:05,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:13:05,707] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:13:05,714] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:06,107] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:06,357] INFO : Plotted number of elements +rank0 [2026-11-15 04:13:07,283] INFO : Plotted jets +rank0 [2026-11-15 04:13:08,571] INFO : Plotted jet ratio +rank0 [2026-11-15 04:13:46,204] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:13:46,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:14:26,504] INFO : Step 14000/500000 rank0 | Train Loss: 10.0977 | Train Loss Unweighted: 2.1143 | Train Loss Components: Classification: 0.0114 | Classification_binary: 1.9458 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0731 | Regression_eta: 0.0045 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4993 | Classification_binary: 1.9458 | Regression_cos_phi: 1.0629 | Regression_energy: 1.4801 | Regression_eta: 1.5740 | Regression_pt: 1.4740 | Regression_sin_phi: 1.0617 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.9284s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.4146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3156.0 | uss_MB: 1958.4 | child_rss_MB: 23100.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:14:26,506] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:14:26,554] INFO : Running validation on rank0 +rank0 [2026-11-15 04:14:26,554] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:14:26,554] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:14:26,657] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:14:26,740] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:14:26,819] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:14:26,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:14:26,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:14:26,960] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:14:27,006] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:14:27,052] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:14:27,121] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:14:27,200] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:14:27,277] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:14:27,324] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:14:27,369] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:14:27,415] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:14:27,460] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:14:27,506] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:14:27,579] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:14:27,583] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:14:27,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:14:27,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:14:27,588] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:14:27,590] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:14:27,592] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:14:27,594] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:14:27,596] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:14:27,597] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:14:27,598] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:14:27,600] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:14:27,601] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:14:27,603] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:14:27,604] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:14:27,606] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:14:28,555] INFO : VALIDATION | Step=14000/500000 | Train Loss=10.0977 | Valid Loss=14.0901 | Train Loss Unweighted=2.1143 | Valid Loss Unweighted=2.2106 | Stale=11 | ETA=686.6m +rank0 [2026-11-15 04:14:28,556] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:14:28,744] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:14:28,753] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,801] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:29,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:29,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:29,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:29,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:29,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:30,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:30,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:30,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:30,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:30,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:30,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:31,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:31,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:31,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:31,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:31,762] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:14:31,769] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:32,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:32,402] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:33,335] INFO : Plotted jets +rank0 [2026-11-15 04:14:34,598] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:38,885] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:14:38,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:20,603] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:15:20,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:50,302] INFO : Step 15000/500000 rank0 | Train Loss: 12.0507 | Train Loss Unweighted: 2.4315 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2255 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0878 | Regression_eta: 0.0049 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.1746 | Classification_binary: 2.2255 | Regression_cos_phi: 1.1062 | Regression_energy: 1.7784 | Regression_eta: 1.7241 | Regression_pt: 1.8050 | Regression_sin_phi: 1.2369 | LR: 3.17e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8617s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0294 | max_loss: 0.0029 | max_backward: 0.0276 | max_optimizer_step: 0.0953 | max_tensorboard_logging: 0.2677 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3178.5 | uss_MB: 1980.8 | child_rss_MB: 23097.5 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:15:50,304] INFO : Running validation on rank0 +rank0 [2026-11-15 04:15:50,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:15:50,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:15:50,415] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:15:50,499] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:15:50,580] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:15:50,631] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:15:50,680] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:15:50,728] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:15:50,777] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:15:50,827] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:15:50,904] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:15:50,985] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:15:51,070] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:15:51,123] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:15:51,177] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:15:51,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:15:51,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:15:51,326] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:15:51,401] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:15:51,405] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:15:51,408] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:15:51,409] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:15:51,410] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:15:51,412] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:15:51,413] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:15:51,415] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:15:51,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:15:51,418] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:15:51,419] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:15:51,421] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:15:51,423] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:15:51,425] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:15:51,426] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:15:51,427] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:15:52,572] INFO : VALIDATION | Step=15000/500000 | Train Loss=12.0507 | Valid Loss=14.2065 | Train Loss Unweighted=2.4315 | Valid Loss Unweighted=2.2272 | Stale=12 | ETA=684.8m +rank0 [2026-11-15 04:15:52,573] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:15:52,707] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:15:52,715] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:52,770] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:15:53,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:15:53,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:15:53,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:15:53,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:15:53,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:15:54,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:15:54,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:15:54,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:15:54,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:15:54,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:15:54,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:15:55,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:15:55,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:15:55,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:15:55,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:15:55,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:15:55,820] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:15:55,827] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:56,225] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:56,480] INFO : Plotted number of elements +rank0 [2026-11-15 04:15:57,418] INFO : Plotted jets +rank0 [2026-11-15 04:15:58,720] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:12,958] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:16:12,958] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:55,687] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:16:55,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:17:16,080] INFO : Step 16000/500000 rank0 | Train Loss: 13.9742 | Train Loss Unweighted: 2.2264 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.0373 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0767 | Regression_eta: 0.0090 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.0930 | Classification_binary: 2.0373 | Regression_cos_phi: 1.1738 | Regression_energy: 1.5530 | Regression_eta: 3.1662 | Regression_pt: 1.5546 | Regression_sin_phi: 2.3964 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7710s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2618 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3198.7 | uss_MB: 2001.1 | child_rss_MB: 23100.6 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 9.1 +rank0 [2026-11-15 04:17:16,082] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:17:16,130] INFO : Running validation on rank0 +rank0 [2026-11-15 04:17:16,130] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:17:16,130] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:17:16,236] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:17:16,322] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:17:16,404] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:17:16,454] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:17:16,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:17:16,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:17:16,602] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:17:16,650] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:17:16,722] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:17:16,804] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:17:16,884] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:17:16,936] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:17:16,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:17:17,037] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:17:17,086] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:17:17,136] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:17:17,215] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:17:17,220] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:17:17,221] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:17:17,224] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:17:17,225] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:17:17,226] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:17:17,228] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:17:17,229] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:17:17,231] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:17:17,232] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:17:17,234] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:17:17,235] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:17:17,238] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:17:17,239] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:17:17,240] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:17:17,242] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:17:18,215] INFO : VALIDATION | Step=16000/500000 | Train Loss=13.9742 | Valid Loss=14.1815 | Train Loss Unweighted=2.2264 | Valid Loss Unweighted=2.2257 | Stale=13 | ETA=683.8m +rank0 [2026-11-15 04:17:18,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:17:18,346] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:17:18,354] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:18,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:17:18,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:17:19,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:17:19,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:17:19,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:17:19,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:17:19,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:17:19,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:17:20,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:17:20,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:17:20,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:17:20,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:17:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:17:20,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:17:21,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:17:21,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:17:21,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:17:21,479] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:17:21,499] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:21,888] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:22,137] INFO : Plotted number of elements +rank0 [2026-11-15 04:17:23,087] INFO : Plotted jets +rank0 [2026-11-15 04:17:24,307] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:46,764] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:17:46,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:28,656] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:18:28,656] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:40,039] INFO : Step 17000/500000 rank0 | Train Loss: 10.1839 | Train Loss Unweighted: 2.3171 | Train Loss Components: Classification: 0.0132 | Classification_binary: 2.1288 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0837 | Regression_eta: 0.0028 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7307 | Classification_binary: 2.1288 | Regression_cos_phi: 0.9944 | Regression_energy: 1.6956 | Regression_eta: 0.9742 | Regression_pt: 1.6632 | Regression_sin_phi: 0.9971 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8304s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0023 | max_backward: 0.0279 | max_optimizer_step: 0.0893 | max_tensorboard_logging: 0.2753 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3140.5 | uss_MB: 1942.8 | child_rss_MB: 23100.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:18:40,040] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:40,041] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:40,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:40,149] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:18:40,236] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:40,316] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:40,366] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:40,413] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:40,459] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:40,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:40,555] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:40,626] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:40,706] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:40,786] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:40,834] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:40,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:40,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:40,976] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:41,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:41,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:41,101] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:41,102] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:41,104] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:41,106] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:41,107] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:41,109] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:41,111] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:41,112] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:41,114] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:41,115] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:41,117] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:41,118] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:41,120] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:41,121] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:41,123] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:42,099] INFO : VALIDATION | Step=17000/500000 | Train Loss=10.1839 | Valid Loss=14.2177 | Train Loss Unweighted=2.3171 | Valid Loss Unweighted=2.2306 | Stale=14 | ETA=682.0m +rank0 [2026-11-15 04:18:42,099] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:18:42,222] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:18:42,231] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:42,293] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:42,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:42,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:43,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:43,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:43,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:43,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:43,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:43,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:44,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:44,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:44,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:44,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:44,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:44,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:45,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:45,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:45,303] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:18:45,310] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:45,690] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:45,939] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:47,442] INFO : Plotted jets +rank0 [2026-11-15 04:18:48,179] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:20,876] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:19:20,876] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:03,226] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:20:03,226] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:05,304] INFO : Step 18000/500000 rank0 | Train Loss: 11.1012 | Train Loss Unweighted: 1.9954 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.8364 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0666 | Regression_eta: 0.0070 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4391 | Classification_binary: 1.8364 | Regression_cos_phi: 1.4656 | Regression_energy: 1.3494 | Regression_eta: 2.4728 | Regression_pt: 1.3427 | Regression_sin_phi: 1.1951 | LR: 3.69e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.8492s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0329 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2586 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3251.0 | uss_MB: 2053.3 | child_rss_MB: 23097.8 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 17.6 +rank0 [2026-11-15 04:20:05,306] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:20:05,364] INFO : Running validation on rank0 +rank0 [2026-11-15 04:20:05,364] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:20:05,364] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:20:05,503] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:20:05,609] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:20:05,695] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:20:05,747] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:20:05,796] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:20:05,846] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:20:05,895] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:20:05,944] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:20:06,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:20:06,095] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:20:06,177] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:20:06,230] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:20:06,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:20:06,335] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:20:06,395] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:20:06,452] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:20:06,542] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:20:06,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:20:06,549] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:20:06,551] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:20:06,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:20:06,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:20:06,558] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:20:06,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:20:06,560] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:20:06,562] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:20:06,564] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:20:06,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:20:06,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:20:06,569] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:20:06,571] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:20:06,574] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:20:07,675] INFO : VALIDATION | Step=18000/500000 | Train Loss=11.1012 | Valid Loss=14.2687 | Train Loss Unweighted=1.9954 | Valid Loss Unweighted=2.2400 | Stale=15 | ETA=681.0m +rank0 [2026-11-15 04:20:07,676] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:20:07,826] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:20:07,834] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:07,898] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:20:08,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:20:08,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:20:08,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:20:09,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:20:09,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:20:09,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:20:09,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:20:09,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:20:10,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:20:10,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:20:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:20:10,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:20:10,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:20:10,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:20:11,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:20:11,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:20:11,233] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:20:11,241] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:11,651] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:11,917] INFO : Plotted number of elements +rank0 [2026-11-15 04:20:12,988] INFO : Plotted jets +rank0 [2026-11-15 04:20:14,574] INFO : Plotted jet ratio +rank0 [2026-11-15 04:20:56,488] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:20:56,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:31,922] INFO : Step 19000/500000 rank0 | Train Loss: 8.6529 | Train Loss Unweighted: 2.0951 | Train Loss Components: Classification: 0.0119 | Classification_binary: 1.9403 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0671 | Regression_eta: 0.0020 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5642 | Classification_binary: 1.9403 | Regression_cos_phi: 0.6455 | Regression_energy: 1.3586 | Regression_eta: 0.6983 | Regression_pt: 1.3818 | Regression_sin_phi: 1.0642 | LR: 3.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.9144s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0274 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0891 | max_tensorboard_logging: 0.3241 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3274.9 | uss_MB: 2077.2 | child_rss_MB: 23101.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:21:31,924] INFO : Running validation on rank0 +rank0 [2026-11-15 04:21:31,924] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:21:31,924] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:21:32,040] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:21:32,127] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:21:32,213] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:21:32,264] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:21:32,316] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:21:32,367] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:21:32,418] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:21:32,470] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:21:32,545] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:21:32,630] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:21:32,712] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:21:32,763] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:21:32,815] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:21:32,864] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:21:32,916] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:21:32,969] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:21:33,048] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:21:33,053] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:21:33,055] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:21:33,056] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:21:33,058] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:21:33,060] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:21:33,061] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:21:33,063] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:21:33,064] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:21:33,066] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:21:33,067] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:21:33,069] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:21:33,071] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:21:33,072] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:21:33,074] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:21:33,075] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:21:34,162] INFO : VALIDATION | Step=19000/500000 | Train Loss=8.6529 | Valid Loss=14.3845 | Train Loss Unweighted=2.0951 | Valid Loss Unweighted=2.2400 | Stale=16 | ETA=680.3m +rank0 [2026-11-15 04:21:34,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:34,310] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:21:34,318] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:34,372] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:34,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:21:35,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:21:35,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:21:35,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:21:35,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:21:35,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:21:35,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:21:36,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:21:36,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:21:36,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:21:36,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:21:36,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:21:36,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:21:37,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:21:37,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:21:37,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:21:37,419] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:21:37,427] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:37,821] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:38,077] INFO : Plotted number of elements +rank0 [2026-11-15 04:21:39,000] INFO : Plotted jets +rank0 [2026-11-15 04:21:40,279] INFO : Plotted jet ratio +rank0 [2026-11-15 04:21:49,283] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:21:49,283] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:31,386] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:22:31,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:58,215] INFO : Step 20000/500000 rank0 | Train Loss: 12.3521 | Train Loss Unweighted: 2.5678 | Train Loss Components: Classification: 0.0115 | Classification_binary: 2.3812 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0794 | Regression_eta: 0.0038 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5066 | Classification_binary: 2.3812 | Regression_cos_phi: 2.1699 | Regression_energy: 1.6086 | Regression_eta: 1.3548 | Regression_pt: 1.6262 | Regression_sin_phi: 1.7047 | LR: 4.07e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8477s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0036 | max_backward: 0.0265 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3214.5 | uss_MB: 2016.8 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:22:58,217] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:22:58,268] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:58,268] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:58,268] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:22:58,374] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:22:58,460] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:22:58,540] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:22:58,591] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:22:58,637] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:22:58,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:22:58,730] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:22:58,778] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:22:58,848] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:22:58,927] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:22:59,007] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:22:59,055] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:22:59,104] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:22:59,151] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:22:59,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:22:59,245] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:22:59,319] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:22:59,324] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:22:59,326] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:22:59,328] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:22:59,329] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:22:59,331] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:22:59,333] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:22:59,334] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:22:59,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:22:59,337] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:22:59,339] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:22:59,340] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:22:59,342] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:22:59,344] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:22:59,345] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:22:59,347] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:00,300] INFO : VALIDATION | Step=20000/500000 | Train Loss=12.3521 | Valid Loss=14.4735 | Train Loss Unweighted=2.5678 | Valid Loss Unweighted=2.2515 | Stale=17 | ETA=679.4m +rank0 [2026-11-15 04:23:00,301] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:00,431] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:23:00,440] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:00,497] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:00,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:01,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:01,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:01,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:01,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:01,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:01,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:02,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:02,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:02,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:02,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:02,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:02,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:03,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:03,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:03,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:03,537] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:23:03,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:03,928] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:04,176] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:05,117] INFO : Plotted jets +rank0 [2026-11-15 04:23:06,430] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:24,240] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:23:24,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:05,898] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:24:05,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:22,336] INFO : Step 21000/500000 rank0 | Train Loss: 9.7641 | Train Loss Unweighted: 2.1715 | Train Loss Components: Classification: 0.0116 | Classification_binary: 2.0036 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0724 | Regression_eta: 0.0036 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5239 | Classification_binary: 2.0036 | Regression_cos_phi: 1.1332 | Regression_energy: 1.4676 | Regression_eta: 1.2712 | Regression_pt: 1.4938 | Regression_sin_phi: 0.8707 | LR: 4.28e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8603s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0846 | max_tensorboard_logging: 0.3201 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3320.2 | uss_MB: 2122.6 | child_rss_MB: 23101.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:24:22,337] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:22,337] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:22,337] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:22,442] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:24:22,524] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:22,604] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:22,655] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:22,701] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:22,747] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:22,795] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:22,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:22,913] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:22,994] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:23,072] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:23,120] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:23,167] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:23,213] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:23,260] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:23,306] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:23,380] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:23,385] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:23,386] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:23,388] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:23,390] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:23,391] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:23,393] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:23,395] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:23,396] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:23,398] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:23,399] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:23,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:23,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:23,404] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:23,406] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:23,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:24,346] INFO : VALIDATION | Step=21000/500000 | Train Loss=9.7641 | Valid Loss=14.4598 | Train Loss Unweighted=2.1715 | Valid Loss Unweighted=2.2505 | Stale=18 | ETA=677.6m +rank0 [2026-11-15 04:24:24,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:24,510] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:24,518] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:24,584] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:25,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:25,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:25,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:25,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:25,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:25,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:26,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:26,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:26,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:26,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:26,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:26,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:27,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:27,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:27,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:27,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:27,628] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:24:27,646] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,020] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,267] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:29,214] INFO : Plotted jets +rank0 [2026-11-15 04:24:30,441] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:56,682] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:24:56,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:38,608] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:25:38,608] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:45,574] INFO : Step 22000/500000 rank0 | Train Loss: 11.2359 | Train Loss Unweighted: 2.0046 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.8538 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0635 | Regression_eta: 0.0084 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6231 | Classification_binary: 1.8538 | Regression_cos_phi: 1.2331 | Regression_energy: 1.2856 | Regression_eta: 2.9837 | Regression_pt: 1.2066 | Regression_sin_phi: 1.0500 | LR: 4.50e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8533s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0022 | max_backward: 0.0279 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.2965 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3343.5 | uss_MB: 2145.8 | child_rss_MB: 23099.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:25:45,576] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:25:45,627] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:45,627] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:45,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:45,742] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:25:45,832] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:45,937] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:45,989] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:46,042] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:46,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:46,149] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:46,203] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:46,279] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:46,361] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:46,442] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:46,496] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:46,549] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:46,601] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:46,653] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:46,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:46,784] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:46,789] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:46,790] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:46,792] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:46,794] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:46,795] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:46,797] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:46,798] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:46,800] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:46,802] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:46,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:46,805] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:46,807] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:46,808] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:46,809] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:46,812] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:47,839] INFO : VALIDATION | Step=22000/500000 | Train Loss=11.2359 | Valid Loss=14.5974 | Train Loss Unweighted=2.0046 | Valid Loss Unweighted=2.2518 | Stale=19 | ETA=675.7m +rank0 [2026-11-15 04:25:47,839] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:48,004] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:48,012] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:48,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:48,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:48,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:48,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:49,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:49,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:49,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:49,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:49,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:49,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:50,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:50,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:50,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:50,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:50,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:50,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:51,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:51,147] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:51,165] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:51,545] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:51,795] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:52,736] INFO : Plotted jets +rank0 [2026-11-15 04:25:54,027] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:30,396] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:26:30,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:10,610] INFO : Step 23000/500000 rank0 | Train Loss: 9.2797 | Train Loss Unweighted: 2.0357 | Train Loss Components: Classification: 0.0118 | Classification_binary: 1.8622 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0770 | Regression_eta: 0.0032 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5527 | Classification_binary: 1.8622 | Regression_cos_phi: 0.7005 | Regression_energy: 1.5593 | Regression_eta: 1.1466 | Regression_pt: 1.5379 | Regression_sin_phi: 0.9205 | LR: 4.73e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9213s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0327 | max_loss: 0.0027 | max_backward: 0.0274 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.3434 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3363.4 | uss_MB: 2165.7 | child_rss_MB: 23098.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 33.6 +rank0 [2026-11-15 04:27:10,612] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:10,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:10,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:10,720] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:27:10,818] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:10,918] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:10,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:11,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:11,086] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:11,138] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:11,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:11,276] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:11,377] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:11,469] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:11,525] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:11,575] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:11,632] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:11,692] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:11,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:11,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:11,870] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:11,871] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:11,875] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:11,877] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:11,879] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:11,881] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:11,882] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:11,885] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:11,887] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:11,888] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:11,890] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:11,893] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:11,896] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:11,898] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:11,899] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:12,863] INFO : VALIDATION | Step=23000/500000 | Train Loss=9.2797 | Valid Loss=14.6583 | Train Loss Unweighted=2.0357 | Valid Loss Unweighted=2.2662 | Stale=20 | ETA=674.4m +rank0 [2026-11-15 04:27:12,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:13,027] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:13,045] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:13,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:13,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:13,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:13,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:14,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:14,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:14,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:14,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:14,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:14,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:15,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:15,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:15,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:15,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:15,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:15,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:16,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:16,138] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:27:16,146] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,530] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,777] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:17,703] INFO : Plotted jets +rank0 [2026-11-15 04:27:18,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:22,396] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:27:22,396] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:06,700] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:28:06,700] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:38,361] INFO : Step 24000/500000 rank0 | Train Loss: 8.3112 | Train Loss Unweighted: 2.1691 | Train Loss Components: Classification: 0.0102 | Classification_binary: 2.0186 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0670 | Regression_eta: 0.0018 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.3436 | Classification_binary: 2.0186 | Regression_cos_phi: 0.8218 | Regression_energy: 1.3575 | Regression_eta: 0.6194 | Regression_pt: 1.3394 | Regression_sin_phi: 0.8108 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8922s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0278 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3114 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3398.6 | uss_MB: 2200.9 | child_rss_MB: 23100.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:28:38,363] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:28:38,412] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:38,412] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:38,412] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:38,515] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:28:38,598] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:38,678] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:38,726] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:38,772] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:38,819] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:38,867] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:38,914] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:38,985] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:39,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:39,145] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:39,193] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:39,240] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:39,286] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:39,333] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,455] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:39,460] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:39,462] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:39,463] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:39,465] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:39,466] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:39,468] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:39,470] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:39,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:39,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:39,475] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:39,476] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:39,478] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:39,480] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:39,481] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:39,483] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:40,434] INFO : VALIDATION | Step=24000/500000 | Train Loss=8.3112 | Valid Loss=14.7293 | Train Loss Unweighted=2.1691 | Valid Loss Unweighted=2.2650 | Stale=21 | ETA=673.9m +rank0 [2026-11-15 04:28:40,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:40,578] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:40,586] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:40,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:41,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:41,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:41,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:41,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:41,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:41,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:42,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:42,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:42,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:42,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:42,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:42,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:43,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:43,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:43,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:43,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:43,690] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:28:43,700] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,083] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,331] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:45,263] INFO : Plotted jets +rank0 [2026-11-15 04:28:46,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:59,617] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:28:59,617] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:41,774] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:29:41,774] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:03,118] INFO : Step 25000/500000 rank0 | Train Loss: 10.5955 | Train Loss Unweighted: 2.1111 | Train Loss Components: Classification: 0.0142 | Classification_binary: 1.9508 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0676 | Regression_eta: 0.0062 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8618 | Classification_binary: 1.9508 | Regression_cos_phi: 1.1716 | Regression_energy: 1.3702 | Regression_eta: 2.2035 | Regression_pt: 1.3413 | Regression_sin_phi: 0.6963 | LR: 5.22e-05 | DataLoad Time: 0.0236s | Model Forward Time: 0.9735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0312 | max_loss: 0.0033 | max_backward: 0.0307 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.3006 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3424.2 | uss_MB: 2226.6 | child_rss_MB: 23104.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:03,119] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:03,119] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:03,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:03,230] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:30:03,320] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:03,405] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:03,459] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:03,509] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:03,556] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:03,604] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,654] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,732] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:03,817] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:03,900] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:03,950] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:04,000] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:04,048] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:04,097] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:04,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:04,229] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:04,234] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:04,235] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:04,237] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:04,239] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:04,240] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:04,242] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:04,243] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:04,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:04,246] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:04,248] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:04,249] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:04,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:04,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:04,254] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:04,256] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:05,303] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.5955 | Valid Loss=14.7204 | Train Loss Unweighted=2.1111 | Valid Loss Unweighted=2.2786 | Stale=22 | ETA=672.4m +rank0 [2026-11-15 04:30:05,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:05,453] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:05,462] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:05,520] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:06,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:06,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:06,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:06,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:06,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:06,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:07,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:07,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:07,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:07,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:07,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:07,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:08,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:08,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:08,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:08,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:08,617] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:08,625] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:09,009] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:09,260] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:10,240] INFO : Plotted jets +rank0 [2026-11-15 04:30:11,688] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:35,189] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:30:35,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:19,737] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:31:19,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:32,267] INFO : Step 26000/500000 rank0 | Train Loss: 8.5751 | Train Loss Unweighted: 2.0508 | Train Loss Components: Classification: 0.0117 | Classification_binary: 1.9005 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0649 | Regression_eta: 0.0016 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5330 | Classification_binary: 1.9005 | Regression_cos_phi: 1.1184 | Regression_energy: 1.3150 | Regression_eta: 0.5664 | Regression_pt: 1.3364 | Regression_sin_phi: 0.8054 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0884 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3446.9 | uss_MB: 2249.2 | child_rss_MB: 23102.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:31:32,269] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:31:32,321] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:32,321] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:32,321] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:32,427] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:31:32,510] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:32,589] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:32,639] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:32,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:32,733] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:32,780] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:32,827] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:32,899] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:32,980] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:33,059] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:33,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:33,154] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:33,201] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:33,249] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:33,295] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:33,370] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:33,375] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:33,377] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:33,379] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:33,380] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:33,382] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:33,384] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:33,385] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:33,387] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:33,389] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:33,390] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:33,393] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:33,394] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:33,395] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:33,397] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:33,398] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:34,370] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.5751 | Valid Loss=14.8880 | Train Loss Unweighted=2.0508 | Valid Loss Unweighted=2.2848 | Stale=23 | ETA=672.3m +rank0 [2026-11-15 04:31:34,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:34,485] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:34,493] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:34,548] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:35,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:35,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:35,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:35,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:35,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:35,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:36,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:36,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:36,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:36,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:36,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:36,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:36,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:37,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:37,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:37,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:37,583] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:37,603] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:38,001] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:38,250] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:39,186] INFO : Plotted jets +rank0 [2026-11-15 04:31:40,502] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:12,106] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:32:12,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:54,733] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:32:54,733] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:57,458] INFO : Step 27000/500000 rank0 | Train Loss: 10.4455 | Train Loss Unweighted: 2.5161 | Train Loss Components: Classification: 0.0131 | Classification_binary: 2.3164 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0884 | Regression_eta: 0.0021 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7264 | Classification_binary: 2.3164 | Regression_cos_phi: 1.0330 | Regression_energy: 1.7907 | Regression_eta: 0.7298 | Regression_pt: 1.8052 | Regression_sin_phi: 1.0440 | LR: 5.74e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9049s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0031 | max_backward: 0.0276 | max_optimizer_step: 0.0932 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3473.0 | uss_MB: 2275.3 | child_rss_MB: 23102.6 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:32:57,460] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:57,460] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:57,460] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:57,577] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:32:57,663] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:57,751] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:57,807] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:57,862] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:57,914] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:57,963] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:58,013] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:58,085] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:58,167] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:58,248] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:58,298] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:58,348] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:58,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:58,445] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:58,493] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:58,570] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:58,575] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:58,576] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:58,578] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:58,580] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:58,582] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:58,583] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:58,585] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:58,586] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:58,589] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:58,590] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:58,591] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:58,593] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:58,595] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:58,596] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:58,598] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:59,621] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.4455 | Valid Loss=15.0035 | Train Loss Unweighted=2.5161 | Valid Loss Unweighted=2.2928 | Stale=24 | ETA=670.9m +rank0 [2026-11-15 04:32:59,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:59,796] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:59,804] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:59,873] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:00,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:00,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:00,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:00,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:01,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:01,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:01,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:01,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:01,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:02,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:02,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:02,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:02,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:02,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:02,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:03,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:03,199] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:33:03,206] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:03,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:03,934] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:05,076] INFO : Plotted jets +rank0 [2026-11-15 04:33:06,614] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:49,707] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:33:49,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:26,613] INFO : Step 28000/500000 rank0 | Train Loss: 12.1931 | Train Loss Unweighted: 2.4226 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.2315 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0815 | Regression_eta: 0.0051 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6162 | Classification_binary: 2.2315 | Regression_cos_phi: 1.7344 | Regression_energy: 1.6506 | Regression_eta: 1.7988 | Regression_pt: 1.6660 | Regression_sin_phi: 1.4956 | LR: 6.01e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8319s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0253 | max_loss: 0.0022 | max_backward: 0.0272 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.3052 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3497.1 | uss_MB: 2299.4 | child_rss_MB: 23099.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:26,614] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:34:26,666] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:26,666] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:26,666] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:26,770] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:34:26,855] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:26,933] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:26,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:27,030] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:27,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:27,123] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:27,169] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:27,239] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:27,319] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:27,398] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:27,447] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:27,494] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:27,540] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:27,587] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:27,634] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:27,708] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:27,713] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:27,715] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:27,716] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:27,718] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:27,719] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:27,721] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:27,723] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:27,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:27,726] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:27,727] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:27,729] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:27,730] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:27,732] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:27,733] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:27,735] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:28,672] INFO : VALIDATION | Step=28000/500000 | Train Loss=12.1931 | Valid Loss=15.0824 | Train Loss Unweighted=2.4226 | Valid Loss Unweighted=2.3018 | Stale=25 | ETA=670.6m +rank0 [2026-11-15 04:34:28,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:28,818] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:28,826] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:28,876] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:29,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:29,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:29,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:29,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:29,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:30,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:30,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:30,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:30,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:30,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:30,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:31,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:31,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:31,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:31,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:31,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:31,921] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:31,929] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:32,324] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:32,570] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:33,502] INFO : Plotted jets +rank0 [2026-11-15 04:34:34,798] INFO : Plotted jet ratio +rank0 [2026-11-15 04:34:42,825] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:34:42,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:25,477] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:35:25,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:53,237] INFO : Step 29000/500000 rank0 | Train Loss: 9.7770 | Train Loss Unweighted: 2.0044 | Train Loss Components: Classification: 0.0120 | Classification_binary: 1.8266 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0785 | Regression_eta: 0.0025 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5756 | Classification_binary: 1.8266 | Regression_cos_phi: 1.1565 | Regression_energy: 1.5909 | Regression_eta: 0.8772 | Regression_pt: 1.5667 | Regression_sin_phi: 1.1835 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9739s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0259 | max_loss: 0.0025 | max_backward: 0.0279 | max_optimizer_step: 0.0961 | max_tensorboard_logging: 0.3296 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3531.1 | uss_MB: 2333.5 | child_rss_MB: 23102.2 | allocated_MB: 170.4 | reserved_MB: 674.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:53,238] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:53,238] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:53,238] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:53,348] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:35:53,434] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:53,520] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:53,575] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:53,629] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:53,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:53,737] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:53,787] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:53,862] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:53,947] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:54,036] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:54,094] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:54,150] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:54,206] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:54,261] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:54,313] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:54,394] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:54,398] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:54,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:54,401] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:54,403] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:54,405] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:54,406] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:54,408] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:54,409] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:54,411] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:54,412] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:54,414] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:54,415] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:54,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:54,418] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:54,420] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:55,492] INFO : VALIDATION | Step=29000/500000 | Train Loss=9.7770 | Valid Loss=15.2448 | Train Loss Unweighted=2.0044 | Valid Loss Unweighted=2.3101 | Stale=26 | ETA=669.6m +rank0 [2026-11-15 04:35:55,492] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:55,633] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:35:55,641] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:55,700] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:56,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:56,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:56,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:56,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:56,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:57,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:57,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:57,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:57,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:57,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:58,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:58,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:58,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:58,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:58,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:58,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:59,244] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:35:59,251] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:59,633] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:59,886] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:00,830] INFO : Plotted jets +rank0 [2026-11-15 04:36:02,133] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:20,472] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:36:20,472] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:03,527] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:37:03,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:20,060] INFO : Step 30000/500000 rank0 | Train Loss: 9.7370 | Train Loss Unweighted: 2.3631 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.2089 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0654 | Regression_eta: 0.0029 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7811 | Classification_binary: 2.2089 | Regression_cos_phi: 1.1780 | Regression_energy: 1.3253 | Regression_eta: 1.0121 | Regression_pt: 1.3344 | Regression_sin_phi: 0.8972 | LR: 6.58e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.9399s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0005 | max_forward: 0.0265 | max_loss: 0.0028 | max_backward: 0.0267 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2927 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3557.1 | uss_MB: 2359.5 | child_rss_MB: 23102.3 | allocated_MB: 170.9 | reserved_MB: 672.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:37:20,062] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:37:20,112] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:20,113] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:20,113] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:20,255] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:37:20,360] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:20,454] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:20,509] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:20,559] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:20,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:20,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:20,712] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:20,787] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:20,869] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:20,951] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:21,004] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:21,057] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:21,112] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:21,168] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,303] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:21,308] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:21,310] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:21,312] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:21,315] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:21,315] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:21,318] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:21,320] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:21,321] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:21,323] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:21,324] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:21,326] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:21,329] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:21,330] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:21,332] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:21,333] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:22,558] INFO : VALIDATION | Step=30000/500000 | Train Loss=9.7370 | Valid Loss=15.3589 | Train Loss Unweighted=2.3631 | Valid Loss Unweighted=2.3177 | Stale=27 | ETA=668.6m +rank0 [2026-11-15 04:37:22,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:22,722] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:22,731] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:22,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:23,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:23,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:23,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:23,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:24,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:24,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:24,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:24,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:24,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:24,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:25,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:25,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:25,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:25,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:25,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:25,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:25,891] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:25,898] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,287] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,539] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:27,471] INFO : Plotted jets +rank0 [2026-11-15 04:37:28,794] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:55,323] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:37:55,323] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:38,088] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:38:38,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:45,697] INFO : Step 31000/500000 rank0 | Train Loss: 11.2127 | Train Loss Unweighted: 2.2649 | Train Loss Components: Classification: 0.0115 | Classification_binary: 2.0693 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0847 | Regression_eta: 0.0042 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5081 | Classification_binary: 2.0693 | Regression_cos_phi: 1.0494 | Regression_energy: 1.7168 | Regression_eta: 1.4792 | Regression_pt: 1.7550 | Regression_sin_phi: 1.6348 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0340 | max_loss: 0.0028 | max_backward: 0.0278 | max_optimizer_step: 0.0935 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3485.9 | uss_MB: 2288.3 | child_rss_MB: 23099.8 | allocated_MB: 170.9 | reserved_MB: 674.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:38:45,699] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:45,699] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:45,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:45,822] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:38:45,931] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:46,022] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:46,078] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:46,130] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:46,185] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:46,239] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:46,295] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:46,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:46,463] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:46,547] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:46,596] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:46,645] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:46,692] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:46,742] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:46,796] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:46,879] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:46,884] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:46,885] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:46,888] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:46,888] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:46,890] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:46,892] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:46,893] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:46,895] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:46,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:46,898] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:46,899] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:46,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:46,904] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:46,904] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:46,907] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:47,972] INFO : VALIDATION | Step=31000/500000 | Train Loss=11.2127 | Valid Loss=15.3007 | Train Loss Unweighted=2.2649 | Valid Loss Unweighted=2.3125 | Stale=28 | ETA=667.2m +rank0 [2026-11-15 04:38:47,972] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:48,115] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:48,124] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,180] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:48,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:48,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:49,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:49,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:49,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:49,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:49,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:49,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:50,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:50,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:50,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:50,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:50,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:50,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:51,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:51,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:51,275] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:51,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:51,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:51,918] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:52,853] INFO : Plotted jets +rank0 [2026-11-15 04:38:54,174] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:30,843] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:39:30,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:11,183] INFO : Step 32000/500000 rank0 | Train Loss: 8.9241 | Train Loss Unweighted: 2.1510 | Train Loss Components: Classification: 0.0113 | Classification_binary: 1.9677 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0849 | Regression_eta: 0.0020 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4862 | Classification_binary: 1.9677 | Regression_cos_phi: 0.6191 | Regression_energy: 1.7191 | Regression_eta: 0.6922 | Regression_pt: 1.6223 | Regression_sin_phi: 0.8175 | LR: 7.19e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0026 | max_backward: 0.0284 | max_optimizer_step: 0.0944 | max_tensorboard_logging: 0.3085 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3592.2 | uss_MB: 2394.6 | child_rss_MB: 23098.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:11,185] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:40:11,235] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:11,235] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:11,235] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:11,347] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:40:11,438] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:11,536] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:11,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:11,654] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:11,708] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:11,759] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:11,814] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:11,891] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:11,988] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:12,085] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:12,147] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:12,205] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:12,261] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:12,321] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:12,393] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:12,495] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:12,500] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:12,503] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:12,505] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:12,507] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:12,510] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:12,511] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:12,514] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:12,515] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:12,517] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:12,521] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:12,523] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:12,524] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:12,526] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:12,529] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:12,531] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:13,585] INFO : VALIDATION | Step=32000/500000 | Train Loss=8.9241 | Valid Loss=15.6161 | Train Loss Unweighted=2.1510 | Valid Loss Unweighted=2.3276 | Stale=29 | ETA=665.9m +rank0 [2026-11-15 04:40:13,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:13,719] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:13,727] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:13,785] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:14,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:14,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:14,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:14,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:15,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:15,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:15,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:15,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:15,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:15,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:15,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:16,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:16,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:16,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:16,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:16,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:17,035] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:40:17,043] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:17,419] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:17,692] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:18,639] INFO : Plotted jets +rank0 [2026-11-15 04:40:20,055] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:23,277] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:40:23,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:07,834] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:41:07,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:38,847] INFO : Step 33000/500000 rank0 | Train Loss: 7.1788 | Train Loss Unweighted: 1.9764 | Train Loss Components: Classification: 0.0088 | Classification_binary: 1.8378 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0623 | Regression_eta: 0.0008 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.1570 | Classification_binary: 1.8378 | Regression_cos_phi: 0.8258 | Regression_energy: 1.2614 | Regression_eta: 0.2905 | Regression_pt: 1.2588 | Regression_sin_phi: 0.5474 | LR: 7.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8810s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0280 | max_optimizer_step: 0.0924 | max_tensorboard_logging: 0.3041 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3616.7 | uss_MB: 2419.0 | child_rss_MB: 23098.8 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:41:38,849] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:38,849] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:38,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:38,957] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:41:39,040] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:39,121] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:39,171] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:39,219] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:39,266] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:39,314] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:39,362] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:39,433] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:39,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:39,593] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:39,642] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:39,690] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:39,737] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:39,785] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:39,832] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:39,907] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:39,912] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:39,913] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:39,915] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:39,921] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:39,922] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:39,924] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:39,925] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:39,927] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:39,928] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:39,930] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:39,931] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:39,933] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:39,935] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:39,936] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:39,938] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:40,927] INFO : VALIDATION | Step=33000/500000 | Train Loss=7.1788 | Valid Loss=15.6315 | Train Loss Unweighted=1.9764 | Valid Loss Unweighted=2.3257 | Stale=30 | ETA=664.9m +rank0 [2026-11-15 04:41:40,927] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:41,073] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:41,081] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:41,148] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:41,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:41,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:42,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:42,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:42,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:42,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:42,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:42,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:42,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:43,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:43,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:43,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:43,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:43,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:43,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:44,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:44,205] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:44,212] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:44,602] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:44,852] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:45,806] INFO : Plotted jets +rank0 [2026-11-15 04:41:47,118] INFO : Plotted jet ratio +rank0 [2026-11-15 04:41:47,577] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:41:47,577] INFO : Training completed. Total time on device 0: 47.096min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..6132a92ae80ba285039ec682df33dd8a12e63101 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/train.log.0 @@ -0,0 +1,2628 @@ +rank0 [2026-11-15 03:54:40,670] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 03:54:40,670] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 03:54:40,670] INFO : Instantiating model +rank0 [2026-11-15 03:54:40,689] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 03:54:40,689] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 03:54:40,690] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 03:54:40,690] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 03:54:40,690] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 03:54:40,690] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 03:54:40,690] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,693] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 03:54:40,693] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,695] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 03:54:40,695] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,697] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 03:54:40,697] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,699] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 03:54:40,699] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,702] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 03:54:40,702] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,704] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 03:54:40,704] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 03:54:40,704] INFO : Initializing output DNNs +rank0 [2026-11-15 03:54:40,705] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,708] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 03:54:40,714] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 03:54:40,714] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 03:54:40,714] INFO : jagged_attention enabled=True +rank0 [2026-11-15 03:54:40,714] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 03:54:40,715] INFO : task_queries enabled=True +rank0 [2026-11-15 03:54:40,715] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 03:54:40,715] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 03:54:40,715] INFO : classification_query parameters: 256 +rank0 [2026-11-15 03:54:40,715] INFO : regression_query parameters: 256 +rank0 [2026-11-15 03:54:40,715] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 03:54:40,715] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 03:54:40,715] INFO : output_mode=elementwise +rank0 [2026-11-15 03:54:40,715] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 03:54:40,715] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 03:54:40,715] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 03:54:40,715] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 03:54:40,715] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 03:54:40,716] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 03:54:40,716] INFO : MLPF __init__ done +rank0 [2026-11-15 03:54:40,716] INFO : Instantiated model +rank0 [2026-11-15 03:54:40,716] INFO : Moving model to device rank=0 +rank0 [2026-11-15 03:54:40,724] INFO : Moved model to device rank=0 +rank0 [2026-11-15 03:54:40,725] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 03:54:40,794] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 03:54:40,794] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 03:54:40,795] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 03:54:40,853] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 03:54:40,853] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 03:54:40,854] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 03:54:40,854] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 03:54:40,854] INFO : Total parameters: 3910930 +rank0 [2026-11-15 03:54:40,855] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 03:54:40,857] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643 +rank0 [2026-11-15 03:54:40,857] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643 +rank0 [2026-11-15 03:54:40,857] INFO : Getting dataloaders +rank0 [2026-11-15 03:54:40,857] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:41,048] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,127] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,176] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,226] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,269] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,311] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,352] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,443] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,479] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,527] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 03:54:41,527] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 03:54:41,528] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 03:54:41,528] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 03:54:41,528] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:54:41,540] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,553] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,564] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,577] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,589] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,601] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,614] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,626] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,638] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,650] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 03:54:41,650] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 03:54:41,650] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 03:54:41,679] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:41,834] INFO : Got dataloaders +rank0 [2026-11-15 03:54:41,835] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 03:54:41,835] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 03:54:41,835] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 03:54:41,839] INFO : Creating train_iterator +rank0 [2026-11-15 03:54:41,839] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:54:41,871] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:54:42,028] INFO : Created train_iterator +rank0 [2026-11-15 03:54:50,567] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9681692945957183, 'Classification': 0.014965717205777765, 'Regression_pt': 0.09860456176102161, 'Regression_eta': 0.005572905512526632, 'Regression_sin_phi': 0.005349073330871761, 'Regression_cos_phi': 0.005312690008431673, 'Regression_energy': 0.0971477759256959}, weights={'Classification_binary': 1.0, 'Classification': 131.51185607910156, 'Regression_pt': 19.96022605895996, 'Regression_eta': 353.16754150390625, 'Regression_sin_phi': 367.9458312988281, 'Regression_cos_phi': 370.4656677246094, 'Regression_energy': 20.259540557861328} +rank0 [2026-11-15 03:55:24,954] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 03:55:24,954] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:55:58,430] INFO : Step 1000/500000 rank0 | Train Loss: 12.4813 | Train Loss Unweighted: 2.3390 | Train Loss Components: Classification: 0.0144 | Classification_binary: 2.1458 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0820 | Regression_eta: 0.0043 | Regression_pt: 0.0827 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8913 | Classification_binary: 2.1458 | Regression_cos_phi: 1.8440 | Regression_energy: 1.6614 | Regression_eta: 1.5243 | Regression_pt: 1.6512 | Regression_sin_phi: 1.7633 | LR: 2.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8341s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0022 | max_backward: 0.0275 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.3369 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2662.7 | uss_MB: 1449.7 | child_rss_MB: 23085.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:55:58,431] INFO : Running validation on rank0 +rank0 [2026-11-15 03:55:58,431] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:55:58,431] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:55:58,558] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 03:55:58,675] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:55:58,732] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:55:58,772] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:55:58,810] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:55:58,844] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:55:58,883] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:55:58,923] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:55:58,958] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:55:59,013] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:55:59,069] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:55:59,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:55:59,155] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:55:59,189] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:55:59,229] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:55:59,268] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:55:59,305] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:55:59,308] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:55:59,309] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:55:59,311] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:55:59,313] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:55:59,314] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:55:59,316] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:55:59,318] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:55:59,319] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:55:59,321] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:55:59,322] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:55:59,324] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:55:59,325] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:55:59,327] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:55:59,329] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:55:59,330] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:56:00,331] INFO : VALIDATION | Step=1000/500000 | Train Loss=12.4813 | Valid Loss=13.7349 | Train Loss Unweighted=2.3390 | Valid Loss Unweighted=2.1596 | Stale=0 | ETA=652.8m +rank0 [2026-11-15 03:56:00,331] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:56:00,480] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:56:00,488] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:00,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:56:01,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:56:01,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:56:01,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:56:01,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:56:01,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:56:01,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:56:01,980] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:56:02,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:56:02,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:56:02,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:56:02,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:56:02,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:56:02,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:56:03,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:56:03,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:56:03,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:56:03,519] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:56:03,537] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:03,929] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:56:04,253] INFO : Plotted number of elements +rank0 [2026-11-15 03:56:05,222] INFO : Plotted jets +rank0 [2026-11-15 03:56:06,454] INFO : Plotted jet ratio +rank0 [2026-11-15 03:56:16,510] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 03:56:16,510] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:00,189] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 03:57:00,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:57:24,089] INFO : Step 2000/500000 rank0 | Train Loss: 15.3531 | Train Loss Unweighted: 2.1627 | Train Loss Components: Classification: 0.0138 | Classification_binary: 1.9078 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1130 | Regression_eta: 0.0066 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8188 | Classification_binary: 1.9078 | Regression_cos_phi: 2.7862 | Regression_energy: 2.2891 | Regression_eta: 2.3318 | Regression_pt: 2.1625 | Regression_sin_phi: 2.0567 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8859s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0791 | max_tensorboard_logging: 0.3387 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2818.1 | uss_MB: 1620.3 | child_rss_MB: 23091.4 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:57:24,091] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 03:57:24,153] INFO : Running validation on rank0 +rank0 [2026-11-15 03:57:24,153] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:57:24,153] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:57:24,271] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 03:57:24,357] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:57:24,435] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:57:24,485] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:57:24,531] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:57:24,577] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:57:24,623] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:57:24,669] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:57:24,738] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:57:24,814] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:57:24,892] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:57:24,939] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:57:24,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:57:25,031] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:57:25,078] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:57:25,124] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:57:25,196] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:57:25,201] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:57:25,203] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:57:25,204] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:57:25,206] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:57:25,208] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:57:25,209] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:57:25,211] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:57:25,212] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:57:25,214] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:57:25,215] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:57:25,217] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:57:25,219] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:57:25,220] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:57:25,222] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:57:25,223] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:57:26,209] INFO : VALIDATION | Step=2000/500000 | Train Loss=15.3531 | Valid Loss=13.7213 | Train Loss Unweighted=2.1627 | Valid Loss Unweighted=2.1631 | Stale=0 | ETA=682.1m +rank0 [2026-11-15 03:57:26,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:57:26,363] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:57:26,378] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:26,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:57:26,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:57:27,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:57:27,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:57:27,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:57:27,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:57:27,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:57:27,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:57:28,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:57:28,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:57:28,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:57:28,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:57:28,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:57:28,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:57:28,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:57:29,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:57:29,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:57:29,391] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:57:29,398] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:29,781] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:57:30,025] INFO : Plotted number of elements +rank0 [2026-11-15 03:57:30,946] INFO : Plotted jets +rank0 [2026-11-15 03:57:32,132] INFO : Plotted jet ratio +rank0 [2026-11-15 03:57:50,559] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 03:57:50,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:31,537] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 03:58:31,537] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 03:58:46,265] INFO : Step 3000/500000 rank0 | Train Loss: 10.8768 | Train Loss Unweighted: 2.0285 | Train Loss Components: Classification: 0.0124 | Classification_binary: 1.8641 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0701 | Regression_eta: 0.0035 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6351 | Classification_binary: 1.8641 | Regression_cos_phi: 1.7712 | Regression_energy: 1.4211 | Regression_eta: 1.2206 | Regression_pt: 1.3831 | Regression_sin_phi: 1.5817 | LR: 2.05e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8762s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0351 | max_loss: 0.0029 | max_backward: 0.0272 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.3022 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2874.8 | uss_MB: 1677.0 | child_rss_MB: 23089.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 03:58:46,266] INFO : Running validation on rank0 +rank0 [2026-11-15 03:58:46,266] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 03:58:46,267] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 03:58:46,388] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 03:58:46,485] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 03:58:46,577] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 03:58:46,634] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 03:58:46,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 03:58:46,739] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 03:58:46,790] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 03:58:46,841] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 03:58:46,915] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 03:58:47,000] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 03:58:47,086] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 03:58:47,137] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 03:58:47,186] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 03:58:47,234] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 03:58:47,282] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 03:58:47,333] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 03:58:47,416] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 03:58:47,421] INFO : plotted pt_target histogram +rank0 [2026-11-15 03:58:47,423] INFO : plotted pt_pred histogram +rank0 [2026-11-15 03:58:47,425] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 03:58:47,427] INFO : plotted eta_target histogram +rank0 [2026-11-15 03:58:47,428] INFO : plotted eta_pred histogram +rank0 [2026-11-15 03:58:47,430] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 03:58:47,432] INFO : plotted sphi_target histogram +rank0 [2026-11-15 03:58:47,433] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 03:58:47,435] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 03:58:47,436] INFO : plotted cphi_target histogram +rank0 [2026-11-15 03:58:47,438] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 03:58:47,440] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 03:58:47,441] INFO : plotted energy_target histogram +rank0 [2026-11-15 03:58:47,443] INFO : plotted energy_pred histogram +rank0 [2026-11-15 03:58:47,444] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 03:58:48,667] INFO : VALIDATION | Step=3000/500000 | Train Loss=10.8768 | Valid Loss=13.7010 | Train Loss Unweighted=2.0285 | Valid Loss Unweighted=2.1665 | Stale=0 | ETA=681.5m +rank0 [2026-11-15 03:58:48,667] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 03:58:48,812] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 03:58:48,821] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:48,887] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 03:58:49,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 03:58:49,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 03:58:49,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 03:58:49,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 03:58:50,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 03:58:50,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 03:58:50,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 03:58:50,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 03:58:50,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 03:58:50,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 03:58:51,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 03:58:51,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 03:58:51,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 03:58:51,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 03:58:51,679] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 03:58:51,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 03:58:51,907] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 03:58:51,914] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:52,290] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 03:58:52,537] INFO : Plotted number of elements +rank0 [2026-11-15 03:58:53,996] INFO : Plotted jets +rank0 [2026-11-15 03:58:54,737] INFO : Plotted jet ratio +rank0 [2026-11-15 03:59:23,612] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 03:59:23,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:04,627] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:00:04,627] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:00:10,553] INFO : Step 4000/500000 rank0 | Train Loss: 16.3001 | Train Loss Unweighted: 2.4321 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.1677 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1137 | Regression_eta: 0.0042 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.3967 | Classification_binary: 2.1677 | Regression_cos_phi: 3.6925 | Regression_energy: 2.3030 | Regression_eta: 1.4852 | Regression_pt: 2.2545 | Regression_sin_phi: 2.0005 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0252 | max_loss: 0.0024 | max_backward: 0.0267 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2904.3 | uss_MB: 1706.5 | child_rss_MB: 23091.3 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:00:10,555] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:00:10,605] INFO : Running validation on rank0 +rank0 [2026-11-15 04:00:10,605] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:00:10,605] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:00:10,713] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:00:10,799] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:00:10,883] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:00:10,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:00:10,983] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:00:11,030] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:00:11,077] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:00:11,125] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:00:11,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:00:11,278] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:00:11,359] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:00:11,407] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:00:11,453] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:00:11,498] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:00:11,543] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:00:11,590] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:00:11,664] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:00:11,669] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:00:11,671] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:00:11,673] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:00:11,674] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:00:11,676] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:00:11,677] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:00:11,679] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:00:11,680] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:00:11,682] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:00:11,684] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:00:11,685] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:00:11,687] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:00:11,689] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:00:11,690] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:00:11,692] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:00:12,632] INFO : VALIDATION | Step=4000/500000 | Train Loss=16.3001 | Valid Loss=13.7493 | Train Loss Unweighted=2.4321 | Valid Loss Unweighted=2.1664 | Stale=1 | ETA=683.6m +rank0 [2026-11-15 04:00:12,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:00:12,779] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:00:12,787] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:12,849] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:00:13,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:00:13,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:00:13,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:00:13,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:00:14,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:00:14,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:00:14,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:00:14,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:00:14,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:00:14,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:00:15,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:00:15,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:00:15,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:00:15,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:00:15,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:00:15,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:00:15,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:00:16,005] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:16,392] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:00:16,641] INFO : Plotted number of elements +rank0 [2026-11-15 04:00:18,243] INFO : Plotted jets +rank0 [2026-11-15 04:00:18,989] INFO : Plotted jet ratio +rank0 [2026-11-15 04:00:56,105] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:00:56,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:01:33,838] INFO : Step 5000/500000 rank0 | Train Loss: 14.1188 | Train Loss Unweighted: 2.1746 | Train Loss Components: Classification: 0.0190 | Classification_binary: 1.9344 | Regression_cos_phi: 0.0036 | Regression_energy: 0.1028 | Regression_eta: 0.0074 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.5041 | Classification_binary: 1.9344 | Regression_cos_phi: 1.3276 | Regression_energy: 2.0820 | Regression_eta: 2.6307 | Regression_pt: 2.0585 | Regression_sin_phi: 1.5815 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0266 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.3572 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2931.1 | uss_MB: 1733.4 | child_rss_MB: 23092.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:01:33,840] INFO : Running validation on rank0 +rank0 [2026-11-15 04:01:33,840] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:01:33,840] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:01:33,945] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:01:34,027] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:01:34,105] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:01:34,156] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:01:34,204] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:01:34,252] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:01:34,299] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:01:34,346] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:01:34,416] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:01:34,494] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:01:34,572] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:01:34,620] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:01:34,666] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:01:34,712] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:01:34,759] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:01:34,806] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:01:34,880] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:01:34,885] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:01:34,887] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:01:34,888] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:01:34,890] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:01:34,892] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:01:34,893] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:01:34,895] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:01:34,896] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:01:34,898] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:01:34,900] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:01:34,901] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:01:34,903] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:01:34,904] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:01:34,906] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:01:34,908] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:01:35,856] INFO : VALIDATION | Step=5000/500000 | Train Loss=14.1188 | Valid Loss=13.7680 | Train Loss Unweighted=2.1746 | Valid Loss Unweighted=2.1706 | Stale=2 | ETA=683.1m +rank0 [2026-11-15 04:01:35,856] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:01:36,023] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:01:36,031] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:36,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:01:36,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:01:36,787] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:01:36,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:01:37,129] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:01:37,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:01:37,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:01:37,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:01:37,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:01:38,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:01:38,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:01:38,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:01:38,507] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:01:38,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:01:38,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:01:39,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:01:39,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:01:39,265] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:01:39,272] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:39,688] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:01:39,981] INFO : Plotted number of elements +rank0 [2026-11-15 04:01:41,028] INFO : Plotted jets +rank0 [2026-11-15 04:01:42,502] INFO : Plotted jet ratio +rank0 [2026-11-15 04:01:47,286] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:01:47,286] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:27,647] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:02:27,647] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:02:55,829] INFO : Step 6000/500000 rank0 | Train Loss: 11.4178 | Train Loss Unweighted: 2.1417 | Train Loss Components: Classification: 0.0128 | Classification_binary: 1.9405 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0874 | Regression_eta: 0.0045 | Regression_pt: 0.0893 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6877 | Classification_binary: 1.9405 | Regression_cos_phi: 1.1390 | Regression_energy: 1.7700 | Regression_eta: 1.6058 | Regression_pt: 1.7833 | Regression_sin_phi: 1.4916 | LR: 2.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8260s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0265 | max_optimizer_step: 0.0876 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2869.3 | uss_MB: 1671.6 | child_rss_MB: 23095.7 | allocated_MB: 170.9 | reserved_MB: 694.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:02:55,831] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:02:55,881] INFO : Running validation on rank0 +rank0 [2026-11-15 04:02:55,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:02:55,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:02:56,015] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:02:56,107] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:02:56,190] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:02:56,242] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:02:56,291] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:02:56,340] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:02:56,391] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:02:56,442] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:02:56,517] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:02:56,599] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:02:56,691] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:02:56,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:02:56,799] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:02:56,856] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:02:56,916] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:02:56,972] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:02:57,059] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:02:57,064] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:02:57,066] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:02:57,068] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:02:57,070] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:02:57,072] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:02:57,074] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:02:57,077] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:02:57,077] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:02:57,079] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:02:57,081] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:02:57,083] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:02:57,085] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:02:57,087] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:02:57,089] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:02:57,091] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:02:58,167] INFO : VALIDATION | Step=6000/500000 | Train Loss=11.4178 | Valid Loss=13.7441 | Train Loss Unweighted=2.1417 | Valid Loss Unweighted=2.1758 | Stale=3 | ETA=681.1m +rank0 [2026-11-15 04:02:58,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:02:58,305] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:02:58,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:02:58,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:02:58,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:02:59,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:02:59,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:02:59,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:02:59,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:02:59,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:02:59,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:03:00,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:03:00,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:03:00,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:03:00,520] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:03:00,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:03:00,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:03:01,025] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:03:01,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:03:01,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:03:01,439] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:03:01,448] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:03:01,825] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:03:02,073] INFO : Plotted number of elements +rank0 [2026-11-15 04:03:03,536] INFO : Plotted jets +rank0 [2026-11-15 04:03:04,267] INFO : Plotted jet ratio +rank0 [2026-11-15 04:03:18,002] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:03:18,003] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:01,973] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:04:01,973] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:04:22,773] INFO : Step 7000/500000 rank0 | Train Loss: 14.2477 | Train Loss Unweighted: 2.1705 | Train Loss Components: Classification: 0.0154 | Classification_binary: 1.9474 | Regression_cos_phi: 0.0076 | Regression_energy: 0.0934 | Regression_eta: 0.0046 | Regression_pt: 0.0966 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.0279 | Classification_binary: 1.9474 | Regression_cos_phi: 2.8146 | Regression_energy: 1.8913 | Regression_eta: 1.6344 | Regression_pt: 1.9289 | Regression_sin_phi: 2.0032 | LR: 2.26e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0300 | max_loss: 0.0027 | max_backward: 0.0276 | max_optimizer_step: 0.0894 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2980.5 | uss_MB: 1782.8 | child_rss_MB: 23093.7 | allocated_MB: 170.4 | reserved_MB: 696.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:04:22,774] INFO : Running validation on rank0 +rank0 [2026-11-15 04:04:22,774] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:04:22,774] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:04:22,883] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:04:22,967] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:04:23,047] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:04:23,096] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:04:23,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:04:23,193] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:04:23,245] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:04:23,297] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:04:23,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:04:23,457] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:04:23,538] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:04:23,587] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:04:23,636] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:04:23,685] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:04:23,738] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:04:23,791] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:04:23,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:04:23,876] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:04:23,878] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:04:23,880] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:04:23,882] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:04:23,883] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:04:23,885] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:04:23,886] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:04:23,888] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:04:23,889] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:04:23,891] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:04:23,893] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:04:23,894] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:04:23,896] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:04:23,897] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:04:23,899] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:04:24,889] INFO : VALIDATION | Step=7000/500000 | Train Loss=14.2477 | Valid Loss=13.7932 | Train Loss Unweighted=2.1705 | Valid Loss Unweighted=2.1804 | Stale=4 | ETA=684.4m +rank0 [2026-11-15 04:04:24,889] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:04:25,027] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:04:25,035] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:25,091] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:04:25,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:04:25,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:04:25,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:04:26,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:04:26,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:04:26,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:04:26,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:04:26,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:04:26,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:04:27,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:04:27,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:04:27,415] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:04:27,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:04:27,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:04:27,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:04:28,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:04:28,151] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:04:28,159] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:28,543] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:04:28,789] INFO : Plotted number of elements +rank0 [2026-11-15 04:04:30,194] INFO : Plotted jets +rank0 [2026-11-15 04:04:30,929] INFO : Plotted jet ratio +rank0 [2026-11-15 04:04:54,328] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:04:54,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:37,912] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:05:37,912] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:05:48,785] INFO : Step 8000/500000 rank0 | Train Loss: 10.7100 | Train Loss Unweighted: 2.2006 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.0107 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0844 | Regression_eta: 0.0027 | Regression_pt: 0.0822 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7790 | Classification_binary: 2.0107 | Regression_cos_phi: 0.9383 | Regression_energy: 1.7090 | Regression_eta: 0.9592 | Regression_pt: 1.6409 | Regression_sin_phi: 1.6729 | LR: 2.34e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.8980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0314 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0882 | max_tensorboard_logging: 0.3182 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3000.4 | uss_MB: 1802.7 | child_rss_MB: 23094.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 25.6 +rank0 [2026-11-15 04:05:48,787] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:05:48,843] INFO : Running validation on rank0 +rank0 [2026-11-15 04:05:48,843] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:05:48,843] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:05:48,972] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:05:49,063] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:05:49,148] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:05:49,206] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:05:49,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:05:49,316] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:05:49,374] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:05:49,428] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:05:49,502] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:05:49,584] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:05:49,669] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:05:49,727] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:05:49,782] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:05:49,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:05:49,894] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:05:49,946] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:05:50,023] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:05:50,028] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:05:50,030] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:05:50,032] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:05:50,034] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:05:50,035] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:05:50,037] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:05:50,039] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:05:50,040] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:05:50,042] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:05:50,044] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:05:50,045] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:05:50,047] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:05:50,049] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:05:50,051] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:05:50,052] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:05:51,162] INFO : VALIDATION | Step=8000/500000 | Train Loss=10.7100 | Valid Loss=13.8837 | Train Loss Unweighted=2.2006 | Valid Loss Unweighted=2.1858 | Stale=5 | ETA=686.1m +rank0 [2026-11-15 04:05:51,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:05:51,319] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:05:51,327] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:51,384] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:05:51,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:05:52,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:05:52,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:05:52,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:05:52,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:05:52,717] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:05:52,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:05:53,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:05:53,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:05:53,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:05:53,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:05:53,700] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:05:53,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:05:54,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:05:54,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:05:54,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:05:54,512] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:05:54,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:54,933] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:05:55,187] INFO : Plotted number of elements +rank0 [2026-11-15 04:05:56,639] INFO : Plotted jets +rank0 [2026-11-15 04:05:57,383] INFO : Plotted jet ratio +rank0 [2026-11-15 04:06:31,090] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:06:31,090] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:14,204] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:07:14,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:07:15,635] INFO : Step 9000/500000 rank0 | Train Loss: 11.9883 | Train Loss Unweighted: 2.3272 | Train Loss Components: Classification: 0.0108 | Classification_binary: 2.1339 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0832 | Regression_eta: 0.0042 | Regression_pt: 0.0854 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4149 | Classification_binary: 2.1339 | Regression_cos_phi: 1.7383 | Regression_energy: 1.6863 | Regression_eta: 1.4843 | Regression_pt: 1.7050 | Regression_sin_phi: 1.8256 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8803s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0278 | max_optimizer_step: 0.0885 | max_tensorboard_logging: 0.3440 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2942.8 | uss_MB: 1745.2 | child_rss_MB: 23095.6 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:07:15,637] INFO : Running validation on rank0 +rank0 [2026-11-15 04:07:15,637] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:07:15,637] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:07:15,756] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:07:15,841] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:07:15,921] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:07:15,971] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:07:16,018] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:07:16,064] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:07:16,111] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:07:16,158] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:07:16,229] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:07:16,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:07:16,393] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:07:16,442] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:07:16,489] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:07:16,536] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:07:16,583] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:07:16,630] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:07:16,703] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:07:16,708] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:07:16,710] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:07:16,712] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:07:16,713] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:07:16,715] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:07:16,716] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:07:16,718] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:07:16,719] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:07:16,721] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:07:16,722] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:07:16,724] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:07:16,726] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:07:16,727] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:07:16,729] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:07:16,730] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:07:17,666] INFO : VALIDATION | Step=9000/500000 | Train Loss=11.9883 | Valid Loss=13.8950 | Train Loss Unweighted=2.3272 | Valid Loss Unweighted=2.1887 | Stale=6 | ETA=687.2m +rank0 [2026-11-15 04:07:17,667] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:07:17,829] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:07:17,837] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:17,897] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:07:18,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:07:18,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:07:18,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:07:18,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:07:19,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:07:19,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:07:19,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:07:19,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:07:19,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:07:19,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:07:20,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:07:20,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:07:20,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:07:20,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:07:20,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:07:20,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:07:20,912] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:07:20,920] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:21,306] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:07:21,563] INFO : Plotted number of elements +rank0 [2026-11-15 04:07:22,969] INFO : Plotted jets +rank0 [2026-11-15 04:07:23,694] INFO : Plotted jet ratio +rank0 [2026-11-15 04:08:07,158] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:08:07,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:08:41,904] INFO : Step 10000/500000 rank0 | Train Loss: 10.8362 | Train Loss Unweighted: 2.0714 | Train Loss Components: Classification: 0.0112 | Classification_binary: 1.8592 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0968 | Regression_eta: 0.0032 | Regression_pt: 0.0941 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4718 | Classification_binary: 1.8592 | Regression_cos_phi: 1.2490 | Regression_energy: 1.9612 | Regression_eta: 1.1132 | Regression_pt: 1.8790 | Regression_sin_phi: 1.3028 | LR: 2.52e-05 | DataLoad Time: 0.0029s | Model Forward Time: 0.8338s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0022 | max_backward: 0.0274 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3052.6 | uss_MB: 1854.9 | child_rss_MB: 23096.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:08:41,906] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:08:41,955] INFO : Running validation on rank0 +rank0 [2026-11-15 04:08:41,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:08:41,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:08:42,078] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:08:42,177] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:08:42,264] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:08:42,323] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:08:42,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:08:42,431] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:08:42,484] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:08:42,538] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:08:42,611] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:08:42,691] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:08:42,771] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:08:42,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:08:42,876] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:08:42,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:08:42,980] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:08:43,029] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:08:43,105] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:08:43,110] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:08:43,112] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:08:43,113] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:08:43,115] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:08:43,117] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:08:43,118] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:08:43,120] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:08:43,121] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:08:43,123] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:08:43,124] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:08:43,126] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:08:43,127] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:08:43,129] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:08:43,131] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:08:43,132] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:08:44,160] INFO : VALIDATION | Step=10000/500000 | Train Loss=10.8362 | Valid Loss=13.9099 | Train Loss Unweighted=2.0714 | Valid Loss Unweighted=2.1926 | Stale=7 | ETA=687.9m +rank0 [2026-11-15 04:08:44,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:08:44,295] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:08:44,316] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:44,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:08:44,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:08:45,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:08:45,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:08:45,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:08:45,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:08:45,764] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:08:45,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:08:46,086] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:08:46,248] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:08:46,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:08:46,589] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:08:46,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:08:46,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:08:47,084] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:08:47,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:08:47,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:08:47,484] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:08:47,492] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:47,882] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:08:48,130] INFO : Plotted number of elements +rank0 [2026-11-15 04:08:49,076] INFO : Plotted jets +rank0 [2026-11-15 04:08:50,351] INFO : Plotted jet ratio +rank0 [2026-11-15 04:09:00,053] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:09:00,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:09:43,184] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:09:43,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:10:08,578] INFO : Step 11000/500000 rank0 | Train Loss: 9.0098 | Train Loss Unweighted: 2.0489 | Train Loss Components: Classification: 0.0101 | Classification_binary: 1.8939 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0668 | Regression_eta: 0.0027 | Regression_pt: 0.0697 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.3313 | Classification_binary: 1.8939 | Regression_cos_phi: 1.2002 | Regression_energy: 1.3539 | Regression_eta: 0.9632 | Regression_pt: 1.3916 | Regression_sin_phi: 0.8758 | LR: 2.63e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0251 | max_loss: 0.0023 | max_backward: 0.0264 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3075.7 | uss_MB: 1878.0 | child_rss_MB: 23094.5 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 04:10:08,580] INFO : Running validation on rank0 +rank0 [2026-11-15 04:10:08,580] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:10:08,580] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:10:08,697] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:10:08,781] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:10:08,861] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:10:08,912] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:10:08,961] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:10:09,009] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:10:09,058] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:10:09,106] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:10:09,177] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:10:09,255] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:10:09,334] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:10:09,381] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:10:09,427] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:10:09,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:10:09,518] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:10:09,564] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:10:09,637] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:10:09,642] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:10:09,644] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:10:09,645] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:10:09,647] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:10:09,648] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:10:09,651] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:10:09,652] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:10:09,653] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:10:09,655] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:10:09,656] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:10:09,658] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:10:09,659] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:10:09,661] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:10:09,662] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:10:09,667] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:10:10,647] INFO : VALIDATION | Step=11000/500000 | Train Loss=9.0098 | Valid Loss=14.0044 | Train Loss Unweighted=2.0489 | Valid Loss Unweighted=2.1949 | Stale=8 | ETA=688.2m +rank0 [2026-11-15 04:10:10,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:10:10,789] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:10:10,798] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:10,855] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:10:11,376] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:10:11,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:10:11,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:10:11,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:10:12,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:10:12,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:10:12,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:10:12,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:10:12,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:10:13,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:10:13,271] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:10:13,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:10:13,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:10:13,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:10:13,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:10:14,079] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:10:14,153] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:10:14,171] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:14,597] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:10:14,870] INFO : Plotted number of elements +rank0 [2026-11-15 04:10:15,883] INFO : Plotted jets +rank0 [2026-11-15 04:10:17,363] INFO : Plotted jet ratio +rank0 [2026-11-15 04:10:36,145] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:10:36,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:19,053] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:11:19,053] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:11:35,081] INFO : Step 12000/500000 rank0 | Train Loss: 7.5518 | Train Loss Unweighted: 1.6791 | Train Loss Components: Classification: 0.0088 | Classification_binary: 1.5423 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0606 | Regression_eta: 0.0022 | Regression_pt: 0.0608 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.1604 | Classification_binary: 1.5423 | Regression_cos_phi: 0.9616 | Regression_energy: 1.2272 | Regression_eta: 0.7621 | Regression_pt: 1.2131 | Regression_sin_phi: 0.6851 | LR: 2.75e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8389s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0832 | max_tensorboard_logging: 0.3018 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3098.7 | uss_MB: 1901.1 | child_rss_MB: 23097.4 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:11:35,083] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:11:35,132] INFO : Running validation on rank0 +rank0 [2026-11-15 04:11:35,132] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:11:35,132] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:11:35,236] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:11:35,319] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:11:35,400] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:11:35,450] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:11:35,496] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:11:35,542] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:11:35,589] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:11:35,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:11:35,707] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:11:35,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:11:35,866] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:11:35,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:11:35,961] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:11:36,007] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:11:36,053] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:11:36,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:11:36,173] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:11:36,178] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:11:36,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:11:36,182] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:11:36,183] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:11:36,185] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:11:36,187] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:11:36,188] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:11:36,190] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:11:36,191] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:11:36,193] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:11:36,195] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:11:36,196] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:11:36,198] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:11:36,200] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:11:36,201] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:11:37,147] INFO : VALIDATION | Step=12000/500000 | Train Loss=7.5518 | Valid Loss=13.9547 | Train Loss Unweighted=1.6791 | Valid Loss Unweighted=2.2042 | Stale=9 | ETA=688.2m +rank0 [2026-11-15 04:11:37,148] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:11:37,306] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:11:37,314] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:37,379] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:11:37,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:11:38,045] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:11:38,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:11:38,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:11:38,541] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:11:38,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:11:38,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:11:39,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:11:39,186] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:11:39,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:11:39,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:11:39,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:11:39,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:11:40,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:11:40,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:11:40,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:11:40,424] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:11:40,431] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:40,819] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:11:41,067] INFO : Plotted number of elements +rank0 [2026-11-15 04:11:42,560] INFO : Plotted jets +rank0 [2026-11-15 04:11:43,291] INFO : Plotted jet ratio +rank0 [2026-11-15 04:12:11,426] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:12:11,426] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:12:53,994] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:12:53,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:13:00,449] INFO : Step 13000/500000 rank0 | Train Loss: 11.5497 | Train Loss Unweighted: 2.2882 | Train Loss Components: Classification: 0.0112 | Classification_binary: 2.1230 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0704 | Regression_eta: 0.0031 | Regression_pt: 0.0696 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4790 | Classification_binary: 2.1230 | Regression_cos_phi: 1.7573 | Regression_energy: 1.4261 | Regression_eta: 1.1067 | Regression_pt: 1.3892 | Regression_sin_phi: 2.2686 | LR: 2.88e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.8832s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0276 | max_optimizer_step: 0.0842 | max_tensorboard_logging: 0.3429 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3129.5 | uss_MB: 1931.8 | child_rss_MB: 23096.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:13:00,450] INFO : Running validation on rank0 +rank0 [2026-11-15 04:13:00,450] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:13:00,451] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:13:00,560] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:13:00,641] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:13:00,721] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:13:00,769] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:13:00,815] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:13:00,861] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:13:00,908] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:13:00,955] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:13:01,025] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:13:01,105] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:13:01,183] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:13:01,230] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:13:01,275] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:13:01,321] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:13:01,367] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:13:01,413] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:13:01,487] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:13:01,492] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:13:01,493] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:13:01,495] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:13:01,496] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:13:01,498] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:13:01,500] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:13:01,501] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:13:01,503] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:13:01,504] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:13:01,506] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:13:01,507] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:13:01,509] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:13:01,510] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:13:01,512] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:13:01,514] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:13:02,496] INFO : VALIDATION | Step=13000/500000 | Train Loss=11.5497 | Valid Loss=14.0858 | Train Loss Unweighted=2.2882 | Valid Loss Unweighted=2.2122 | Stale=10 | ETA=687.2m +rank0 [2026-11-15 04:13:02,496] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:13:02,656] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:13:02,664] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:02,717] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:13:03,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:13:03,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:13:03,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:13:03,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:13:03,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:13:04,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:13:04,174] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:13:04,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:13:04,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:13:04,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:13:04,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:13:04,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:13:05,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:13:05,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:13:05,466] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:13:05,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:13:05,707] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:13:05,714] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:06,107] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:13:06,357] INFO : Plotted number of elements +rank0 [2026-11-15 04:13:07,283] INFO : Plotted jets +rank0 [2026-11-15 04:13:08,571] INFO : Plotted jet ratio +rank0 [2026-11-15 04:13:46,204] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:13:46,204] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:14:26,504] INFO : Step 14000/500000 rank0 | Train Loss: 10.0977 | Train Loss Unweighted: 2.1143 | Train Loss Components: Classification: 0.0114 | Classification_binary: 1.9458 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0731 | Regression_eta: 0.0045 | Regression_pt: 0.0738 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4993 | Classification_binary: 1.9458 | Regression_cos_phi: 1.0629 | Regression_energy: 1.4801 | Regression_eta: 1.5740 | Regression_pt: 1.4740 | Regression_sin_phi: 1.0617 | LR: 3.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.9284s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.4146 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3156.0 | uss_MB: 1958.4 | child_rss_MB: 23100.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:14:26,506] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:14:26,554] INFO : Running validation on rank0 +rank0 [2026-11-15 04:14:26,554] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:14:26,554] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:14:26,657] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:14:26,740] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:14:26,819] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:14:26,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:14:26,915] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:14:26,960] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:14:27,006] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:14:27,052] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:14:27,121] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:14:27,200] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:14:27,277] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:14:27,324] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:14:27,369] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:14:27,415] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:14:27,460] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:14:27,506] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:14:27,579] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:14:27,583] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:14:27,585] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:14:27,588] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:14:27,588] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:14:27,590] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:14:27,592] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:14:27,594] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:14:27,596] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:14:27,597] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:14:27,598] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:14:27,600] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:14:27,601] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:14:27,603] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:14:27,604] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:14:27,606] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:14:28,555] INFO : VALIDATION | Step=14000/500000 | Train Loss=10.0977 | Valid Loss=14.0901 | Train Loss Unweighted=2.1143 | Valid Loss Unweighted=2.2106 | Stale=11 | ETA=686.6m +rank0 [2026-11-15 04:14:28,556] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:14:28,744] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:14:28,753] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:28,801] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:14:29,249] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:14:29,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:14:29,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:14:29,741] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:14:29,902] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:14:30,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:14:30,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:14:30,387] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:14:30,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:14:30,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:14:30,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:14:31,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:14:31,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:14:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:14:31,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:14:31,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:14:31,762] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:14:31,769] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:32,151] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:14:32,402] INFO : Plotted number of elements +rank0 [2026-11-15 04:14:33,335] INFO : Plotted jets +rank0 [2026-11-15 04:14:34,598] INFO : Plotted jet ratio +rank0 [2026-11-15 04:14:38,885] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:14:38,886] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:20,603] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:15:20,603] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:15:50,302] INFO : Step 15000/500000 rank0 | Train Loss: 12.0507 | Train Loss Unweighted: 2.4315 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2255 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0878 | Regression_eta: 0.0049 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.1746 | Classification_binary: 2.2255 | Regression_cos_phi: 1.1062 | Regression_energy: 1.7784 | Regression_eta: 1.7241 | Regression_pt: 1.8050 | Regression_sin_phi: 1.2369 | LR: 3.17e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8617s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0294 | max_loss: 0.0029 | max_backward: 0.0276 | max_optimizer_step: 0.0953 | max_tensorboard_logging: 0.2677 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3178.5 | uss_MB: 1980.8 | child_rss_MB: 23097.5 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:15:50,304] INFO : Running validation on rank0 +rank0 [2026-11-15 04:15:50,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:15:50,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:15:50,415] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:15:50,499] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:15:50,580] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:15:50,631] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:15:50,680] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:15:50,728] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:15:50,777] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:15:50,827] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:15:50,904] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:15:50,985] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:15:51,070] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:15:51,123] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:15:51,177] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:15:51,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:15:51,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:15:51,326] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:15:51,401] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:15:51,405] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:15:51,408] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:15:51,409] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:15:51,410] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:15:51,412] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:15:51,413] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:15:51,415] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:15:51,416] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:15:51,418] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:15:51,419] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:15:51,421] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:15:51,423] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:15:51,425] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:15:51,426] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:15:51,427] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:15:52,572] INFO : VALIDATION | Step=15000/500000 | Train Loss=12.0507 | Valid Loss=14.2065 | Train Loss Unweighted=2.4315 | Valid Loss Unweighted=2.2272 | Stale=12 | ETA=684.8m +rank0 [2026-11-15 04:15:52,573] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:15:52,707] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:15:52,715] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:52,770] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:15:53,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:15:53,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:15:53,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:15:53,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:15:53,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:15:54,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:15:54,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:15:54,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:15:54,607] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:15:54,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:15:54,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:15:55,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:15:55,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:15:55,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:15:55,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:15:55,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:15:55,820] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:15:55,827] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:56,225] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:15:56,480] INFO : Plotted number of elements +rank0 [2026-11-15 04:15:57,418] INFO : Plotted jets +rank0 [2026-11-15 04:15:58,720] INFO : Plotted jet ratio +rank0 [2026-11-15 04:16:12,958] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:16:12,958] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:16:55,687] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:16:55,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:17:16,080] INFO : Step 16000/500000 rank0 | Train Loss: 13.9742 | Train Loss Unweighted: 2.2264 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.0373 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0767 | Regression_eta: 0.0090 | Regression_pt: 0.0779 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 2.0930 | Classification_binary: 2.0373 | Regression_cos_phi: 1.1738 | Regression_energy: 1.5530 | Regression_eta: 3.1662 | Regression_pt: 1.5546 | Regression_sin_phi: 2.3964 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7710s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0796 | max_tensorboard_logging: 0.2618 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3198.7 | uss_MB: 2001.1 | child_rss_MB: 23100.6 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 9.1 +rank0 [2026-11-15 04:17:16,082] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:17:16,130] INFO : Running validation on rank0 +rank0 [2026-11-15 04:17:16,130] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:17:16,130] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:17:16,236] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:17:16,322] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:17:16,404] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:17:16,454] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:17:16,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:17:16,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:17:16,602] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:17:16,650] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:17:16,722] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:17:16,804] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:17:16,884] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:17:16,936] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:17:16,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:17:17,037] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:17:17,086] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:17:17,136] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:17:17,215] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:17:17,220] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:17:17,221] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:17:17,224] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:17:17,225] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:17:17,226] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:17:17,228] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:17:17,229] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:17:17,231] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:17:17,232] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:17:17,234] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:17:17,235] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:17:17,238] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:17:17,239] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:17:17,240] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:17:17,242] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:17:18,215] INFO : VALIDATION | Step=16000/500000 | Train Loss=13.9742 | Valid Loss=14.1815 | Train Loss Unweighted=2.2264 | Valid Loss Unweighted=2.2257 | Stale=13 | ETA=683.8m +rank0 [2026-11-15 04:17:18,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:17:18,346] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:17:18,354] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:18,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:17:18,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:17:19,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:17:19,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:17:19,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:17:19,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:17:19,762] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:17:19,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:17:20,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:17:20,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:17:20,405] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:17:20,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:17:20,746] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:17:20,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:17:21,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:17:21,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:17:21,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:17:21,479] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:17:21,499] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:21,888] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:17:22,137] INFO : Plotted number of elements +rank0 [2026-11-15 04:17:23,087] INFO : Plotted jets +rank0 [2026-11-15 04:17:24,307] INFO : Plotted jet ratio +rank0 [2026-11-15 04:17:46,764] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:17:46,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:28,656] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:18:28,656] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:18:40,039] INFO : Step 17000/500000 rank0 | Train Loss: 10.1839 | Train Loss Unweighted: 2.3171 | Train Loss Components: Classification: 0.0132 | Classification_binary: 2.1288 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0837 | Regression_eta: 0.0028 | Regression_pt: 0.0833 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7307 | Classification_binary: 2.1288 | Regression_cos_phi: 0.9944 | Regression_energy: 1.6956 | Regression_eta: 0.9742 | Regression_pt: 1.6632 | Regression_sin_phi: 0.9971 | LR: 3.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8304s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0023 | max_backward: 0.0279 | max_optimizer_step: 0.0893 | max_tensorboard_logging: 0.2753 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3140.5 | uss_MB: 1942.8 | child_rss_MB: 23100.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:18:40,040] INFO : Running validation on rank0 +rank0 [2026-11-15 04:18:40,041] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:18:40,041] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:18:40,149] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:18:40,236] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:18:40,316] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:18:40,366] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:18:40,413] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:18:40,459] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:18:40,507] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:18:40,555] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:18:40,626] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:18:40,706] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:18:40,786] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:18:40,834] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:18:40,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:18:40,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:18:40,976] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:18:41,023] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:18:41,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:18:41,101] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:18:41,102] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:18:41,104] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:18:41,106] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:18:41,107] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:18:41,109] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:18:41,111] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:18:41,112] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:18:41,114] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:18:41,115] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:18:41,117] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:18:41,118] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:18:41,120] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:18:41,121] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:18:41,123] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:18:42,099] INFO : VALIDATION | Step=17000/500000 | Train Loss=10.1839 | Valid Loss=14.2177 | Train Loss Unweighted=2.3171 | Valid Loss Unweighted=2.2306 | Stale=14 | ETA=682.0m +rank0 [2026-11-15 04:18:42,099] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:18:42,222] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:18:42,231] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:42,293] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:18:42,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:18:42,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:18:43,092] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:18:43,256] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:18:43,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:18:43,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:18:43,748] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:18:43,922] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:18:44,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:18:44,254] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:18:44,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:18:44,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:18:44,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:18:44,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:18:45,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:18:45,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:18:45,303] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:18:45,310] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:45,690] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:18:45,939] INFO : Plotted number of elements +rank0 [2026-11-15 04:18:47,442] INFO : Plotted jets +rank0 [2026-11-15 04:18:48,179] INFO : Plotted jet ratio +rank0 [2026-11-15 04:19:20,876] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:19:20,876] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:03,226] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:20:03,226] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:20:05,304] INFO : Step 18000/500000 rank0 | Train Loss: 11.1012 | Train Loss Unweighted: 1.9954 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.8364 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0666 | Regression_eta: 0.0070 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4391 | Classification_binary: 1.8364 | Regression_cos_phi: 1.4656 | Regression_energy: 1.3494 | Regression_eta: 2.4728 | Regression_pt: 1.3427 | Regression_sin_phi: 1.1951 | LR: 3.69e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.8492s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0329 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2586 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3251.0 | uss_MB: 2053.3 | child_rss_MB: 23097.8 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 17.6 +rank0 [2026-11-15 04:20:05,306] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:20:05,364] INFO : Running validation on rank0 +rank0 [2026-11-15 04:20:05,364] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:20:05,364] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:20:05,503] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:20:05,609] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:20:05,695] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:20:05,747] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:20:05,796] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:20:05,846] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:20:05,895] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:20:05,944] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:20:06,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:20:06,095] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:20:06,177] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:20:06,230] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:20:06,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:20:06,335] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:20:06,395] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:20:06,452] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:20:06,542] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:20:06,547] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:20:06,549] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:20:06,551] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:20:06,553] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:20:06,555] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:20:06,558] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:20:06,559] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:20:06,560] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:20:06,562] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:20:06,564] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:20:06,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:20:06,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:20:06,569] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:20:06,571] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:20:06,574] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:20:07,675] INFO : VALIDATION | Step=18000/500000 | Train Loss=11.1012 | Valid Loss=14.2687 | Train Loss Unweighted=1.9954 | Valid Loss Unweighted=2.2400 | Stale=15 | ETA=681.0m +rank0 [2026-11-15 04:20:07,676] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:20:07,826] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:20:07,834] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:07,898] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:20:08,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:20:08,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:20:08,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:20:09,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:20:09,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:20:09,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:20:09,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:20:09,819] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:20:10,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:20:10,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:20:10,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:20:10,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:20:10,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:20:10,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:20:11,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:20:11,172] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:20:11,233] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:20:11,241] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:11,651] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:20:11,917] INFO : Plotted number of elements +rank0 [2026-11-15 04:20:12,988] INFO : Plotted jets +rank0 [2026-11-15 04:20:14,574] INFO : Plotted jet ratio +rank0 [2026-11-15 04:20:56,488] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:20:56,488] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:31,922] INFO : Step 19000/500000 rank0 | Train Loss: 8.6529 | Train Loss Unweighted: 2.0951 | Train Loss Components: Classification: 0.0119 | Classification_binary: 1.9403 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0671 | Regression_eta: 0.0020 | Regression_pt: 0.0692 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5642 | Classification_binary: 1.9403 | Regression_cos_phi: 0.6455 | Regression_energy: 1.3586 | Regression_eta: 0.6983 | Regression_pt: 1.3818 | Regression_sin_phi: 1.0642 | LR: 3.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.9144s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0274 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0891 | max_tensorboard_logging: 0.3241 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3274.9 | uss_MB: 2077.2 | child_rss_MB: 23101.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:21:31,924] INFO : Running validation on rank0 +rank0 [2026-11-15 04:21:31,924] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:21:31,924] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:21:32,040] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:21:32,127] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:21:32,213] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:21:32,264] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:21:32,316] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:21:32,367] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:21:32,418] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:21:32,470] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:21:32,545] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:21:32,630] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:21:32,712] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:21:32,763] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:21:32,815] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:21:32,864] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:21:32,916] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:21:32,969] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:21:33,048] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:21:33,053] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:21:33,055] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:21:33,056] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:21:33,058] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:21:33,060] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:21:33,061] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:21:33,063] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:21:33,064] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:21:33,066] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:21:33,067] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:21:33,069] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:21:33,071] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:21:33,072] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:21:33,074] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:21:33,075] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:21:34,162] INFO : VALIDATION | Step=19000/500000 | Train Loss=8.6529 | Valid Loss=14.3845 | Train Loss Unweighted=2.0951 | Valid Loss Unweighted=2.2400 | Stale=16 | ETA=680.3m +rank0 [2026-11-15 04:21:34,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:34,310] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:21:34,318] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:34,372] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:34,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:21:35,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:21:35,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:21:35,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:21:35,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:21:35,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:21:35,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:21:36,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:21:36,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:21:36,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:21:36,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:21:36,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:21:36,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:21:37,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:21:37,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:21:37,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:21:37,419] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:21:37,427] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:37,821] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:21:38,077] INFO : Plotted number of elements +rank0 [2026-11-15 04:21:39,000] INFO : Plotted jets +rank0 [2026-11-15 04:21:40,279] INFO : Plotted jet ratio +rank0 [2026-11-15 04:21:49,283] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:21:49,283] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:31,386] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:22:31,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:22:58,215] INFO : Step 20000/500000 rank0 | Train Loss: 12.3521 | Train Loss Unweighted: 2.5678 | Train Loss Components: Classification: 0.0115 | Classification_binary: 2.3812 | Regression_cos_phi: 0.0059 | Regression_energy: 0.0794 | Regression_eta: 0.0038 | Regression_pt: 0.0815 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5066 | Classification_binary: 2.3812 | Regression_cos_phi: 2.1699 | Regression_energy: 1.6086 | Regression_eta: 1.3548 | Regression_pt: 1.6262 | Regression_sin_phi: 1.7047 | LR: 4.07e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8477s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0036 | max_backward: 0.0265 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3214.5 | uss_MB: 2016.8 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:22:58,217] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:22:58,268] INFO : Running validation on rank0 +rank0 [2026-11-15 04:22:58,268] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:22:58,268] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:22:58,374] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:22:58,460] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:22:58,540] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:22:58,591] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:22:58,637] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:22:58,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:22:58,730] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:22:58,778] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:22:58,848] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:22:58,927] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:22:59,007] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:22:59,055] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:22:59,104] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:22:59,151] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:22:59,198] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:22:59,245] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:22:59,319] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:22:59,324] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:22:59,326] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:22:59,328] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:22:59,329] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:22:59,331] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:22:59,333] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:22:59,334] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:22:59,336] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:22:59,337] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:22:59,339] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:22:59,340] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:22:59,342] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:22:59,344] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:22:59,345] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:22:59,347] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:00,300] INFO : VALIDATION | Step=20000/500000 | Train Loss=12.3521 | Valid Loss=14.4735 | Train Loss Unweighted=2.5678 | Valid Loss Unweighted=2.2515 | Stale=17 | ETA=679.4m +rank0 [2026-11-15 04:23:00,301] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:00,431] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:23:00,440] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:00,497] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:00,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:01,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:01,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:01,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:01,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:01,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:01,972] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:02,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:02,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:02,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:02,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:02,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:02,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:03,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:03,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:03,433] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:03,537] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:23:03,547] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:03,928] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:04,176] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:05,117] INFO : Plotted jets +rank0 [2026-11-15 04:23:06,430] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:24,240] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:23:24,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:05,898] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:24:05,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:22,336] INFO : Step 21000/500000 rank0 | Train Loss: 9.7641 | Train Loss Unweighted: 2.1715 | Train Loss Components: Classification: 0.0116 | Classification_binary: 2.0036 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0724 | Regression_eta: 0.0036 | Regression_pt: 0.0748 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5239 | Classification_binary: 2.0036 | Regression_cos_phi: 1.1332 | Regression_energy: 1.4676 | Regression_eta: 1.2712 | Regression_pt: 1.4938 | Regression_sin_phi: 0.8707 | LR: 4.28e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8603s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0846 | max_tensorboard_logging: 0.3201 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3320.2 | uss_MB: 2122.6 | child_rss_MB: 23101.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:24:22,337] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:22,337] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:22,337] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:22,442] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:24:22,524] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:22,604] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:22,655] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:22,701] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:22,747] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:22,795] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:22,842] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:22,913] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:22,994] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:23,072] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:23,120] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:23,167] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:23,213] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:23,260] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:23,306] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:23,380] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:23,385] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:23,386] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:23,388] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:23,390] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:23,391] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:23,393] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:23,395] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:23,396] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:23,398] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:23,399] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:23,401] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:23,403] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:23,404] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:23,406] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:23,408] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:24,346] INFO : VALIDATION | Step=21000/500000 | Train Loss=9.7641 | Valid Loss=14.4598 | Train Loss Unweighted=2.1715 | Valid Loss Unweighted=2.2505 | Stale=18 | ETA=677.6m +rank0 [2026-11-15 04:24:24,347] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:24,510] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:24,518] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:24,584] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:25,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:25,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:25,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:25,584] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:25,745] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:25,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:26,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:26,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:26,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:26,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:26,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:26,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:27,057] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:27,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:27,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:27,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:27,628] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:24:27,646] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,020] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,267] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:29,214] INFO : Plotted jets +rank0 [2026-11-15 04:24:30,441] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:56,682] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:24:56,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:38,608] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:25:38,608] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:45,574] INFO : Step 22000/500000 rank0 | Train Loss: 11.2359 | Train Loss Unweighted: 2.0046 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.8538 | Regression_cos_phi: 0.0033 | Regression_energy: 0.0635 | Regression_eta: 0.0084 | Regression_pt: 0.0605 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6231 | Classification_binary: 1.8538 | Regression_cos_phi: 1.2331 | Regression_energy: 1.2856 | Regression_eta: 2.9837 | Regression_pt: 1.2066 | Regression_sin_phi: 1.0500 | LR: 4.50e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8533s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0022 | max_backward: 0.0279 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.2965 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3343.5 | uss_MB: 2145.8 | child_rss_MB: 23099.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:25:45,576] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:25:45,627] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:45,627] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:45,627] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:45,742] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:25:45,832] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:45,937] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:45,989] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:46,042] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:46,096] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:46,149] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:46,203] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:46,279] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:46,361] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:46,442] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:46,496] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:46,549] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:46,601] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:46,653] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:46,704] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:46,784] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:46,789] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:46,790] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:46,792] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:46,794] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:46,795] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:46,797] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:46,798] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:46,800] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:46,802] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:46,804] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:46,805] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:46,807] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:46,808] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:46,809] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:46,812] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:47,839] INFO : VALIDATION | Step=22000/500000 | Train Loss=11.2359 | Valid Loss=14.5974 | Train Loss Unweighted=2.0046 | Valid Loss Unweighted=2.2518 | Stale=19 | ETA=675.7m +rank0 [2026-11-15 04:25:47,839] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:48,004] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:48,012] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:48,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:48,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:48,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:48,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:49,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:49,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:49,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:49,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:49,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:49,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:50,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:50,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:50,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:50,592] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:50,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:50,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:51,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:51,147] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:51,165] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:51,545] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:51,795] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:52,736] INFO : Plotted jets +rank0 [2026-11-15 04:25:54,027] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:30,396] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:26:30,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:10,610] INFO : Step 23000/500000 rank0 | Train Loss: 9.2797 | Train Loss Unweighted: 2.0357 | Train Loss Components: Classification: 0.0118 | Classification_binary: 1.8622 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0770 | Regression_eta: 0.0032 | Regression_pt: 0.0770 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5527 | Classification_binary: 1.8622 | Regression_cos_phi: 0.7005 | Regression_energy: 1.5593 | Regression_eta: 1.1466 | Regression_pt: 1.5379 | Regression_sin_phi: 0.9205 | LR: 4.73e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9213s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0327 | max_loss: 0.0027 | max_backward: 0.0274 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.3434 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3363.4 | uss_MB: 2165.7 | child_rss_MB: 23098.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 33.6 +rank0 [2026-11-15 04:27:10,612] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:10,612] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:10,612] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:10,720] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:27:10,818] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:10,918] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:10,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:11,036] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:11,086] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:11,138] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:11,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:11,276] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:11,377] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:11,469] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:11,525] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:11,575] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:11,632] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:11,692] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:11,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:11,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:11,870] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:11,871] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:11,875] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:11,877] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:11,879] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:11,881] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:11,882] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:11,885] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:11,887] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:11,888] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:11,890] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:11,893] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:11,896] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:11,898] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:11,899] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:12,863] INFO : VALIDATION | Step=23000/500000 | Train Loss=9.2797 | Valid Loss=14.6583 | Train Loss Unweighted=2.0357 | Valid Loss Unweighted=2.2662 | Stale=20 | ETA=674.4m +rank0 [2026-11-15 04:27:12,863] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:13,027] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:13,045] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:13,109] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:13,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:13,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:13,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:14,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:14,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:14,423] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:14,583] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:14,743] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:14,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:15,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:15,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:15,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:15,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:15,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:15,898] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:16,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:16,138] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:27:16,146] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,530] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,777] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:17,703] INFO : Plotted jets +rank0 [2026-11-15 04:27:18,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:27:22,396] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:27:22,396] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:06,700] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:28:06,700] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:38,361] INFO : Step 24000/500000 rank0 | Train Loss: 8.3112 | Train Loss Unweighted: 2.1691 | Train Loss Components: Classification: 0.0102 | Classification_binary: 2.0186 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0670 | Regression_eta: 0.0018 | Regression_pt: 0.0671 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.3436 | Classification_binary: 2.0186 | Regression_cos_phi: 0.8218 | Regression_energy: 1.3575 | Regression_eta: 0.6194 | Regression_pt: 1.3394 | Regression_sin_phi: 0.8108 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8922s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0278 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3114 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3398.6 | uss_MB: 2200.9 | child_rss_MB: 23100.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:28:38,363] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:28:38,412] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:38,412] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:38,412] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:38,515] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:28:38,598] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:38,678] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:38,726] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:38,772] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:38,819] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:38,867] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:38,914] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:38,985] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:39,065] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:39,145] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:39,193] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:39,240] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:39,286] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:39,333] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,380] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,455] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:39,460] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:39,462] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:39,463] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:39,465] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:39,466] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:39,468] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:39,470] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:39,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:39,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:39,475] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:39,476] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:39,478] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:39,480] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:39,481] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:39,483] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:40,434] INFO : VALIDATION | Step=24000/500000 | Train Loss=8.3112 | Valid Loss=14.7293 | Train Loss Unweighted=2.1691 | Valid Loss Unweighted=2.2650 | Stale=21 | ETA=673.9m +rank0 [2026-11-15 04:28:40,434] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:40,578] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:40,586] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:40,641] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:41,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:41,295] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:41,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:41,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:41,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:41,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:42,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:42,273] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:42,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:42,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:42,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:42,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:43,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:43,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:43,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:43,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:43,690] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:28:43,700] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,083] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,331] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:45,263] INFO : Plotted jets +rank0 [2026-11-15 04:28:46,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:59,617] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:28:59,617] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:41,774] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:29:41,774] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:03,118] INFO : Step 25000/500000 rank0 | Train Loss: 10.5955 | Train Loss Unweighted: 2.1111 | Train Loss Components: Classification: 0.0142 | Classification_binary: 1.9508 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0676 | Regression_eta: 0.0062 | Regression_pt: 0.0672 | Regression_sin_phi: 0.0019 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.8618 | Classification_binary: 1.9508 | Regression_cos_phi: 1.1716 | Regression_energy: 1.3702 | Regression_eta: 2.2035 | Regression_pt: 1.3413 | Regression_sin_phi: 0.6963 | LR: 5.22e-05 | DataLoad Time: 0.0236s | Model Forward Time: 0.9735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0312 | max_loss: 0.0033 | max_backward: 0.0307 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.3006 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3424.2 | uss_MB: 2226.6 | child_rss_MB: 23104.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:03,119] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:03,119] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:03,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:03,230] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:30:03,320] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:03,405] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:03,459] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:03,509] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:03,556] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:03,604] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,654] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,732] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:03,817] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:03,900] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:03,950] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:04,000] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:04,048] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:04,097] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:04,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:04,229] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:04,234] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:04,235] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:04,237] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:04,239] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:04,240] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:04,242] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:04,243] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:04,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:04,246] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:04,248] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:04,249] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:04,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:04,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:04,254] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:04,256] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:05,303] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.5955 | Valid Loss=14.7204 | Train Loss Unweighted=2.1111 | Valid Loss Unweighted=2.2786 | Stale=22 | ETA=672.4m +rank0 [2026-11-15 04:30:05,303] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:05,453] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:05,462] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:05,520] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:06,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:06,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:06,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:06,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:06,730] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:06,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:07,053] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:07,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:07,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:07,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:07,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:07,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:08,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:08,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:08,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:08,526] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:08,617] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:08,625] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:09,009] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:09,260] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:10,240] INFO : Plotted jets +rank0 [2026-11-15 04:30:11,688] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:35,189] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:30:35,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:19,737] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:31:19,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:32,267] INFO : Step 26000/500000 rank0 | Train Loss: 8.5751 | Train Loss Unweighted: 2.0508 | Train Loss Components: Classification: 0.0117 | Classification_binary: 1.9005 | Regression_cos_phi: 0.0030 | Regression_energy: 0.0649 | Regression_eta: 0.0016 | Regression_pt: 0.0670 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5330 | Classification_binary: 1.9005 | Regression_cos_phi: 1.1184 | Regression_energy: 1.3150 | Regression_eta: 0.5664 | Regression_pt: 1.3364 | Regression_sin_phi: 0.8054 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0884 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3446.9 | uss_MB: 2249.2 | child_rss_MB: 23102.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:31:32,269] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:31:32,321] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:32,321] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:32,321] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:32,427] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:31:32,510] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:32,589] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:32,639] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:32,686] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:32,733] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:32,780] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:32,827] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:32,899] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:32,980] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:33,059] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:33,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:33,154] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:33,201] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:33,249] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:33,295] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:33,370] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:33,375] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:33,377] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:33,379] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:33,380] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:33,382] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:33,384] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:33,385] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:33,387] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:33,389] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:33,390] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:33,393] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:33,394] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:33,395] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:33,397] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:33,398] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:34,370] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.5751 | Valid Loss=14.8880 | Train Loss Unweighted=2.0508 | Valid Loss Unweighted=2.2848 | Stale=23 | ETA=672.3m +rank0 [2026-11-15 04:31:34,370] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:34,485] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:34,493] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:34,548] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:35,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:35,195] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:35,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:35,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:35,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:35,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:36,011] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:36,173] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:36,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:36,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:36,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:36,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:36,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:37,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:37,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:37,464] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:37,583] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:37,603] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:38,001] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:38,250] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:39,186] INFO : Plotted jets +rank0 [2026-11-15 04:31:40,502] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:12,106] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:32:12,106] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:54,733] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:32:54,733] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:57,458] INFO : Step 27000/500000 rank0 | Train Loss: 10.4455 | Train Loss Unweighted: 2.5161 | Train Loss Components: Classification: 0.0131 | Classification_binary: 2.3164 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0884 | Regression_eta: 0.0021 | Regression_pt: 0.0904 | Regression_sin_phi: 0.0028 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7264 | Classification_binary: 2.3164 | Regression_cos_phi: 1.0330 | Regression_energy: 1.7907 | Regression_eta: 0.7298 | Regression_pt: 1.8052 | Regression_sin_phi: 1.0440 | LR: 5.74e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9049s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0031 | max_backward: 0.0276 | max_optimizer_step: 0.0932 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3473.0 | uss_MB: 2275.3 | child_rss_MB: 23102.6 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:32:57,460] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:57,460] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:57,460] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:57,577] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:32:57,663] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:57,751] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:57,807] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:57,862] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:57,914] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:57,963] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:58,013] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:58,085] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:58,167] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:58,248] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:58,298] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:58,348] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:58,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:58,445] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:58,493] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:58,570] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:58,575] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:58,576] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:58,578] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:58,580] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:58,582] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:58,583] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:58,585] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:58,586] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:58,589] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:58,590] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:58,591] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:58,593] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:58,595] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:58,596] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:58,598] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:59,621] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.4455 | Valid Loss=15.0035 | Train Loss Unweighted=2.5161 | Valid Loss Unweighted=2.2928 | Stale=24 | ETA=670.9m +rank0 [2026-11-15 04:32:59,621] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:59,796] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:59,804] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:59,873] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:00,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:00,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:00,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:00,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:01,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:01,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:01,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:01,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:01,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:02,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:02,257] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:02,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:02,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:02,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:02,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:03,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:03,199] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:33:03,206] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:03,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:03,934] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:05,076] INFO : Plotted jets +rank0 [2026-11-15 04:33:06,614] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:49,707] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 04:33:49,707] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:26,613] INFO : Step 28000/500000 rank0 | Train Loss: 12.1931 | Train Loss Unweighted: 2.4226 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.2315 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0815 | Regression_eta: 0.0051 | Regression_pt: 0.0835 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.6162 | Classification_binary: 2.2315 | Regression_cos_phi: 1.7344 | Regression_energy: 1.6506 | Regression_eta: 1.7988 | Regression_pt: 1.6660 | Regression_sin_phi: 1.4956 | LR: 6.01e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8319s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0253 | max_loss: 0.0022 | max_backward: 0.0272 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.3052 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3497.1 | uss_MB: 2299.4 | child_rss_MB: 23099.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:26,614] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 04:34:26,666] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:26,666] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:26,666] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:26,770] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 04:34:26,855] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:26,933] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:26,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:27,030] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:27,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:27,123] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:27,169] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:27,239] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:27,319] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:27,398] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:27,447] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:27,494] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:27,540] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:27,587] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:27,634] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:27,708] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:27,713] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:27,715] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:27,716] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:27,718] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:27,719] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:27,721] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:27,723] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:27,724] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:27,726] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:27,727] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:27,729] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:27,730] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:27,732] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:27,733] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:27,735] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:28,672] INFO : VALIDATION | Step=28000/500000 | Train Loss=12.1931 | Valid Loss=15.0824 | Train Loss Unweighted=2.4226 | Valid Loss Unweighted=2.3018 | Stale=25 | ETA=670.6m +rank0 [2026-11-15 04:34:28,672] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:28,818] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:28,826] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:28,876] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:29,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:29,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:29,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:29,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:29,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:30,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:30,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:30,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:30,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:30,800] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:30,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:31,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:31,285] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:31,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:31,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:31,772] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:31,921] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:31,929] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:32,324] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:32,570] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:33,502] INFO : Plotted jets +rank0 [2026-11-15 04:34:34,798] INFO : Plotted jet ratio +rank0 [2026-11-15 04:34:42,825] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 04:34:42,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:25,477] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 04:35:25,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:53,237] INFO : Step 29000/500000 rank0 | Train Loss: 9.7770 | Train Loss Unweighted: 2.0044 | Train Loss Components: Classification: 0.0120 | Classification_binary: 1.8266 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0785 | Regression_eta: 0.0025 | Regression_pt: 0.0785 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5756 | Classification_binary: 1.8266 | Regression_cos_phi: 1.1565 | Regression_energy: 1.5909 | Regression_eta: 0.8772 | Regression_pt: 1.5667 | Regression_sin_phi: 1.1835 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9739s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0259 | max_loss: 0.0025 | max_backward: 0.0279 | max_optimizer_step: 0.0961 | max_tensorboard_logging: 0.3296 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3531.1 | uss_MB: 2333.5 | child_rss_MB: 23102.2 | allocated_MB: 170.4 | reserved_MB: 674.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:53,238] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:53,238] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:53,238] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:53,348] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 04:35:53,434] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:53,520] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:53,575] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:53,629] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:53,683] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:53,737] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:53,787] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:53,862] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:53,947] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:54,036] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:54,094] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:54,150] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:54,206] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:54,261] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:54,313] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:54,394] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:54,398] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:54,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:54,401] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:54,403] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:54,405] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:54,406] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:54,408] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:54,409] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:54,411] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:54,412] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:54,414] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:54,415] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:54,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:54,418] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:54,420] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:55,492] INFO : VALIDATION | Step=29000/500000 | Train Loss=9.7770 | Valid Loss=15.2448 | Train Loss Unweighted=2.0044 | Valid Loss Unweighted=2.3101 | Stale=26 | ETA=669.6m +rank0 [2026-11-15 04:35:55,492] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:55,633] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:35:55,641] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:55,700] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:56,247] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:56,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:56,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:56,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:56,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:57,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:57,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:57,524] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:57,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:57,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:58,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:58,262] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:58,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:58,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:58,815] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:58,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:59,244] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:35:59,251] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:59,633] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:59,886] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:00,830] INFO : Plotted jets +rank0 [2026-11-15 04:36:02,133] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:20,472] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 04:36:20,472] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:03,527] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 04:37:03,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:20,060] INFO : Step 30000/500000 rank0 | Train Loss: 9.7370 | Train Loss Unweighted: 2.3631 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.2089 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0654 | Regression_eta: 0.0029 | Regression_pt: 0.0669 | Regression_sin_phi: 0.0024 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.7811 | Classification_binary: 2.2089 | Regression_cos_phi: 1.1780 | Regression_energy: 1.3253 | Regression_eta: 1.0121 | Regression_pt: 1.3344 | Regression_sin_phi: 0.8972 | LR: 6.58e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.9399s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0005 | max_forward: 0.0265 | max_loss: 0.0028 | max_backward: 0.0267 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2927 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3557.1 | uss_MB: 2359.5 | child_rss_MB: 23102.3 | allocated_MB: 170.9 | reserved_MB: 672.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:37:20,062] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 04:37:20,112] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:20,113] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:20,113] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:20,255] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 04:37:20,360] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:20,454] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:20,509] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:20,559] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:20,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:20,661] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:20,712] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:20,787] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:20,869] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:20,951] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:21,004] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:21,057] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:21,112] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:21,168] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,303] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:21,308] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:21,310] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:21,312] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:21,315] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:21,315] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:21,318] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:21,320] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:21,321] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:21,323] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:21,324] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:21,326] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:21,329] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:21,330] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:21,332] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:21,333] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:22,558] INFO : VALIDATION | Step=30000/500000 | Train Loss=9.7370 | Valid Loss=15.3589 | Train Loss Unweighted=2.3631 | Valid Loss Unweighted=2.3177 | Stale=27 | ETA=668.6m +rank0 [2026-11-15 04:37:22,558] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:22,722] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:22,731] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:22,791] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:23,338] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:23,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:23,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:23,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:24,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:24,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:24,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:24,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:24,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:24,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:25,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:25,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:25,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:25,480] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:25,641] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:25,788] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:25,891] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:25,898] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,287] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,539] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:27,471] INFO : Plotted jets +rank0 [2026-11-15 04:37:28,794] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:55,323] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 04:37:55,323] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:38,088] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 04:38:38,088] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:45,697] INFO : Step 31000/500000 rank0 | Train Loss: 11.2127 | Train Loss Unweighted: 2.2649 | Train Loss Components: Classification: 0.0115 | Classification_binary: 2.0693 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0847 | Regression_eta: 0.0042 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.5081 | Classification_binary: 2.0693 | Regression_cos_phi: 1.0494 | Regression_energy: 1.7168 | Regression_eta: 1.4792 | Regression_pt: 1.7550 | Regression_sin_phi: 1.6348 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9303s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0340 | max_loss: 0.0028 | max_backward: 0.0278 | max_optimizer_step: 0.0935 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3485.9 | uss_MB: 2288.3 | child_rss_MB: 23099.8 | allocated_MB: 170.9 | reserved_MB: 674.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:38:45,699] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:45,699] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:45,699] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:45,822] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 04:38:45,931] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:46,022] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:46,078] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:46,130] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:46,185] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:46,239] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:46,295] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:46,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:46,463] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:46,547] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:46,596] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:46,645] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:46,692] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:46,742] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:46,796] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:46,879] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:46,884] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:46,885] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:46,888] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:46,888] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:46,890] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:46,892] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:46,893] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:46,895] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:46,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:46,898] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:46,899] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:46,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:46,904] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:46,904] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:46,907] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:47,972] INFO : VALIDATION | Step=31000/500000 | Train Loss=11.2127 | Valid Loss=15.3007 | Train Loss Unweighted=2.2649 | Valid Loss Unweighted=2.3125 | Stale=28 | ETA=667.2m +rank0 [2026-11-15 04:38:47,972] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:48,115] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:48,124] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,180] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:48,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:48,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:49,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:49,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:49,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:49,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:49,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:49,859] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:50,020] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:50,179] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:50,339] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:50,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:50,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:50,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:51,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:51,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:51,275] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:51,282] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:51,667] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:51,918] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:52,853] INFO : Plotted jets +rank0 [2026-11-15 04:38:54,174] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:30,843] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 04:39:30,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:11,183] INFO : Step 32000/500000 rank0 | Train Loss: 8.9241 | Train Loss Unweighted: 2.1510 | Train Loss Components: Classification: 0.0113 | Classification_binary: 1.9677 | Regression_cos_phi: 0.0017 | Regression_energy: 0.0849 | Regression_eta: 0.0020 | Regression_pt: 0.0813 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.4862 | Classification_binary: 1.9677 | Regression_cos_phi: 0.6191 | Regression_energy: 1.7191 | Regression_eta: 0.6922 | Regression_pt: 1.6223 | Regression_sin_phi: 0.8175 | LR: 7.19e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0026 | max_backward: 0.0284 | max_optimizer_step: 0.0944 | max_tensorboard_logging: 0.3085 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3592.2 | uss_MB: 2394.6 | child_rss_MB: 23098.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:11,185] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 04:40:11,235] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:11,235] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:11,235] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:11,347] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 04:40:11,438] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:11,536] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:11,596] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:11,654] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:11,708] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:11,759] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:11,814] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:11,891] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:11,988] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:12,085] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:12,147] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:12,205] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:12,261] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:12,321] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:12,393] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:12,495] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:12,500] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:12,503] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:12,505] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:12,507] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:12,510] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:12,511] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:12,514] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:12,515] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:12,517] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:12,521] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:12,523] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:12,524] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:12,526] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:12,529] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:12,531] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:13,585] INFO : VALIDATION | Step=32000/500000 | Train Loss=8.9241 | Valid Loss=15.6161 | Train Loss Unweighted=2.1510 | Valid Loss Unweighted=2.3276 | Stale=29 | ETA=665.9m +rank0 [2026-11-15 04:40:13,585] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:13,719] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:13,727] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:13,785] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:14,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:14,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:14,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:14,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:15,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:15,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:15,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:15,488] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:15,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:15,817] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:15,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:16,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:16,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:16,479] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:16,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:16,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:17,035] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:40:17,043] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:17,419] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:17,692] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:18,639] INFO : Plotted jets +rank0 [2026-11-15 04:40:20,055] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:23,277] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 04:40:23,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:07,834] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 04:41:07,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:38,847] INFO : Step 33000/500000 rank0 | Train Loss: 7.1788 | Train Loss Unweighted: 1.9764 | Train Loss Components: Classification: 0.0088 | Classification_binary: 1.8378 | Regression_cos_phi: 0.0022 | Regression_energy: 0.0623 | Regression_eta: 0.0008 | Regression_pt: 0.0631 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 131.5119 | Classification_binary: 1.0000 | Regression_cos_phi: 370.4657 | Regression_energy: 20.2595 | Regression_eta: 353.1675 | Regression_pt: 19.9602 | Regression_sin_phi: 367.9458 | Task Weighted Losses: Classification: 1.1570 | Classification_binary: 1.8378 | Regression_cos_phi: 0.8258 | Regression_energy: 1.2614 | Regression_eta: 0.2905 | Regression_pt: 1.2588 | Regression_sin_phi: 0.5474 | LR: 7.51e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8810s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0280 | max_optimizer_step: 0.0924 | max_tensorboard_logging: 0.3041 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3616.7 | uss_MB: 2419.0 | child_rss_MB: 23098.8 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:41:38,849] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:38,849] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:38,849] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:38,957] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 04:41:39,040] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:39,121] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:39,171] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:39,219] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:39,266] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:39,314] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:39,362] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:39,433] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:39,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:39,593] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:39,642] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:39,690] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:39,737] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:39,785] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:39,832] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:39,907] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:39,912] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:39,913] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:39,915] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:39,921] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:39,922] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:39,924] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:39,925] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:39,927] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:39,928] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:39,930] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:39,931] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:39,933] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:39,935] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:39,936] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:39,938] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:40,927] INFO : VALIDATION | Step=33000/500000 | Train Loss=7.1788 | Valid Loss=15.6315 | Train Loss Unweighted=1.9764 | Valid Loss Unweighted=2.3257 | Stale=30 | ETA=664.9m +rank0 [2026-11-15 04:41:40,927] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:41,073] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:41,081] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:41,148] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:41,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:41,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:42,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:42,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:42,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:42,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:42,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:42,831] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:42,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:43,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:43,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:43,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:43,645] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:43,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:43,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:44,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:44,205] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:44,212] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:44,602] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_20mm_seed12345_20261115_035431_565643/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:44,852] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:45,806] INFO : Plotted jets +rank0 [2026-11-15 04:41:47,118] INFO : Plotted jet ratio +rank0 [2026-11-15 04:41:47,577] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 04:41:47,577] INFO : Training completed. Total time on device 0: 47.096min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..fe0d37aa75791e9e8974aa69dccdfe76f882eb53 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d8ba0a97b1bea8916d5ce9ee9cf4fa5c2d9b7e05814d439bfa7b18a647df8d +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-32000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-32000.pth new file mode 100644 index 0000000000000000000000000000000000000000..0a78585af4090f24bad3843890e743b1e99d20b3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-32000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7cc923f39089fd4ad1745262d86aa52b30660c4c299f0ade5b82ae34aca5ed +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..51a95d8f04c443e0c0a86d0619f4cacaabfd8a22 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 2.041351318359375, "Classification": 0.014840541407465935, "Regression_pt": 0.10634798556566238, "Regression_eta": 0.003672950668260455, "Regression_sin_phi": 0.008352015167474747, "Regression_cos_phi": 0.005124746821820736, "Regression_energy": 0.10693880170583725, "Total": 14.814077377319336, "Total_unweighted": 2.2866283596958965}, "train_group": {}, "valid": {"Classification_binary": 1.9802271127700806, "Classification": 0.014233849011361599, "Regression_pt": 0.08686845004558563, "Regression_eta": 0.007081709336489439, "Regression_sin_phi": 0.0055422550067305565, "Regression_cos_phi": 0.005297459661960602, "Regression_energy": 0.08726855367422104, "Total": 13.980196952819824, "Total_unweighted": 2.1865193895064294, "metrics/particle/class_1/efficiency": 0.9511423110961914, "metrics/particle/class_1/pid_accuracy": 0.9873155951499939, "metrics/particle/class_2/efficiency": 0.5343642830848694, "metrics/particle/class_2/pid_accuracy": 0.848874568939209, "metrics/particle/class_3/efficiency": 0.8380334377288818, "metrics/particle/class_3/pid_accuracy": 0.9909141659736633, "metrics/particle/class_4/efficiency": 0.8847457766532898, "metrics/particle/class_4/pid_accuracy": 0.6130267977714539, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7683615684509277, "metrics/particle/count/bias_mean": 0.5720000267028809, "metrics/particle/count/mae": 3.131999969482422, "metrics/particle/count/prediction_mean": 98.76399993896484, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02453034371137619, "metrics/particle/event/energy_response_mean": 1.0027318000793457, "metrics/particle/event/met_abs_error": 4.220258712768555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02340313047170639, "metrics/particle/event/scalar_pt_response_mean": 1.0040003061294556, "metrics/particle/event/vector_pt_closure": 0.02652418427169323, "metrics/particle/matched/delta_eta_abs_mean": 0.008571739308536053, "metrics/particle/matched/delta_phi_abs_mean": 0.00983366183936596, "metrics/particle/matched/delta_r_mean": 0.014601396396756172, "metrics/particle/matched/energy_relative_abs_mean": 0.08636488020420074, "metrics/particle/matched/pid_accuracy": 0.976404070854187, "metrics/particle/matched/pt_relative_abs_mean": 0.08111432194709778, "metrics/particle/matching/duplicate_fraction": 0.025110363960266113, "metrics/particle/matching/efficiency": 0.8718429207801819, "metrics/particle/matching/f1": 0.8693109154701233, "metrics/particle/matching/purity": 0.8667935729026794, "metrics/particle/matching/target_coverage_dr0p05": 0.8503747582435608, "metrics/particle/matching/target_coverage_dr0p10": 0.8993808031082153, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.767964780330658, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9026397466659546}, "step_train_time": 0.7862205505371094, "step_valid_time": 1.805754920002073, "step_total_time": 78.82138133049011, "elapsed_training_loop_seconds": 78.82138133049011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024067566844752, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059122136562404615, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05056553559547571, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494344644045243, "step_test_time": 3.212434389977716, "step_plot_time": 2.810341548989527, "step_validation_cycle_time": 8.23516279796604} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..2eb47802eab52bf56e3783a2f45e588dd3a00adb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 2.1766414642333984, "Classification": 0.017626985907554626, "Regression_pt": 0.10721458494663239, "Regression_eta": 0.009634038433432579, "Regression_sin_phi": 0.0046149250119924545, "Regression_cos_phi": 0.006531080696731806, "Regression_energy": 0.10675421357154846, "Total": 16.40306854248047, "Total_unweighted": 2.4290172928012908}, "train_group": {}, "valid": {"Classification_binary": 2.004456043243408, "Classification": 0.014461314305663109, "Regression_pt": 0.08980666100978851, "Regression_eta": 0.007113903295248747, "Regression_sin_phi": 0.005609935615211725, "Regression_cos_phi": 0.005391885060817003, "Regression_energy": 0.09052908420562744, "Total": 14.234747886657715, "Total_unweighted": 2.2173688267357647, "metrics/particle/class_1/efficiency": 0.950457751750946, "metrics/particle/class_1/pid_accuracy": 0.988206684589386, "metrics/particle/class_2/efficiency": 0.5360824465751648, "metrics/particle/class_2/pid_accuracy": 0.8376068472862244, "metrics/particle/class_3/efficiency": 0.8338973522186279, "metrics/particle/class_3/pid_accuracy": 0.9910945892333984, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.6070038676261902, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.7942857146263123, "metrics/particle/count/bias_mean": 0.36399999260902405, "metrics/particle/count/mae": 3.059999942779541, "metrics/particle/count/prediction_mean": 98.55599975585938, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024420596659183502, "metrics/particle/event/energy_response_mean": 1.0009222030639648, "metrics/particle/event/met_abs_error": 3.967288017272949, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02378656156361103, "metrics/particle/event/scalar_pt_response_mean": 1.0030102729797363, "metrics/particle/event/vector_pt_closure": 0.02596377767622471, "metrics/particle/matched/delta_eta_abs_mean": 0.008772015571594238, "metrics/particle/matched/delta_phi_abs_mean": 0.00978308729827404, "metrics/particle/matched/delta_r_mean": 0.014702469110488892, "metrics/particle/matched/energy_relative_abs_mean": 0.0861804261803627, "metrics/particle/matched/pid_accuracy": 0.9766243696212769, "metrics/particle/matched/pt_relative_abs_mean": 0.08050492405891418, "metrics/particle/matching/duplicate_fraction": 0.025488046929240227, "metrics/particle/matching/efficiency": 0.869602382183075, "metrics/particle/matching/f1": 0.8679935932159424, "metrics/particle/matching/purity": 0.8663907051086426, "metrics/particle/matching/target_coverage_dr0p05": 0.8479713201522827, "metrics/particle/matching/target_coverage_dr0p10": 0.8979957699775696, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7667427062988281, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9003992080688477}, "step_train_time": 0.8455567359924316, "step_valid_time": 1.9851558189839125, "step_total_time": 834.834389925003, "elapsed_training_loop_seconds": 834.834389925003, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030714988345102, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05898812120640784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9560878243512974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959279038718291, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576807730756415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04072096128170895, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9552736982643525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536821059646784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9560878243512974, "step_test_time": 3.537136416998692, "step_plot_time": 2.8411714190151542, "step_validation_cycle_time": 8.736866156978067} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..67a053baef295aa041638647967b5d0c363a1bee --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 1.901618242263794, "Classification": 0.01872626505792141, "Regression_pt": 0.10867380350828171, "Regression_eta": 0.006955866236239672, "Regression_sin_phi": 0.005728445947170258, "Regression_cos_phi": 0.0048833987675607204, "Regression_energy": 0.104110486805439, "Total": 15.141571044921875, "Total_unweighted": 2.1506965085864067}, "train_group": {}, "valid": {"Classification_binary": 2.0048913955688477, "Classification": 0.014454797841608524, "Regression_pt": 0.08898241072893143, "Regression_eta": 0.007199612911790609, "Regression_sin_phi": 0.005696424748748541, "Regression_cos_phi": 0.005464366637170315, "Regression_energy": 0.08980393409729004, "Total": 14.293453216552734, "Total_unweighted": 2.216492942534387, "metrics/particle/class_1/efficiency": 0.9525113105773926, "metrics/particle/class_1/pid_accuracy": 0.9882321357727051, "metrics/particle/class_2/efficiency": 0.5326460599899292, "metrics/particle/class_2/pid_accuracy": 0.8354838490486145, "metrics/particle/class_3/efficiency": 0.8336153626441956, "metrics/particle/class_3/pid_accuracy": 0.9904149770736694, "metrics/particle/class_4/efficiency": 0.8610169291496277, "metrics/particle/class_4/pid_accuracy": 0.6062992215156555, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.800000011920929, "metrics/particle/count/bias_mean": 0.41200000047683716, "metrics/particle/count/mae": 3.068000078201294, "metrics/particle/count/prediction_mean": 98.60399627685547, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024921737611293793, "metrics/particle/event/energy_response_mean": 1.0007637739181519, "metrics/particle/event/met_abs_error": 4.237891674041748, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024509776383638382, "metrics/particle/event/scalar_pt_response_mean": 1.003048300743103, "metrics/particle/event/vector_pt_closure": 0.02671937458217144, "metrics/particle/matched/delta_eta_abs_mean": 0.008412051014602184, "metrics/particle/matched/delta_phi_abs_mean": 0.009896667674183846, "metrics/particle/matched/delta_r_mean": 0.014508089050650597, "metrics/particle/matched/energy_relative_abs_mean": 0.08572763204574585, "metrics/particle/matched/pid_accuracy": 0.9764034152030945, "metrics/particle/matched/pt_relative_abs_mean": 0.08020847290754318, "metrics/particle/matching/duplicate_fraction": 0.025029409676790237, "metrics/particle/matching/efficiency": 0.8700912594795227, "metrics/particle/matching/f1": 0.868269681930542, "metrics/particle/matching/purity": 0.8664557337760925, "metrics/particle/matching/target_coverage_dr0p05": 0.8479306101799011, "metrics/particle/matching/target_coverage_dr0p10": 0.8979957699775696, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7686980366706848, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9018657207489014}, "step_train_time": 0.8439476490020752, "step_valid_time": 1.9919311809935607, "step_total_time": 919.9400148391724, "elapsed_training_loop_seconds": 919.9400148391724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001411788516153, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06014620703967688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9534264803725881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559706470980653, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546968687541639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044029352901934625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533022014676451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9520319786808794, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9534264803725881, "step_test_time": 3.193592926952988, "step_plot_time": 2.7538944630068727, "step_validation_cycle_time": 8.304491703980602} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..62ad262e7e3e38f2340151ee88174f204f222938 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 1.971591591835022, "Classification": 0.0160012599080801, "Regression_pt": 0.07211077958345413, "Regression_eta": 0.006977198179811239, "Regression_sin_phi": 0.005146552808582783, "Regression_cos_phi": 0.006964093539863825, "Regression_energy": 0.07140421867370605, "Total": 14.049140930175781, "Total_unweighted": 2.15019569452852}, "train_group": {}, "valid": {"Classification_binary": 2.011288642883301, "Classification": 0.014530080370604992, "Regression_pt": 0.08945179730653763, "Regression_eta": 0.007235725410282612, "Regression_sin_phi": 0.0055661736987531185, "Regression_cos_phi": 0.005411282181739807, "Regression_energy": 0.09024375677108765, "Total": 14.268489837646484, "Total_unweighted": 2.2237274586223066, "metrics/particle/class_1/efficiency": 0.9514845609664917, "metrics/particle/class_1/pid_accuracy": 0.9888489246368408, "metrics/particle/class_2/efficiency": 0.5355097651481628, "metrics/particle/class_2/pid_accuracy": 0.8385026454925537, "metrics/particle/class_3/efficiency": 0.8352133631706238, "metrics/particle/class_3/pid_accuracy": 0.9905458688735962, "metrics/particle/class_4/efficiency": 0.8610169291496277, "metrics/particle/class_4/pid_accuracy": 0.5984252095222473, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.7885714173316956, "metrics/particle/count/bias_mean": 0.6159999966621399, "metrics/particle/count/mae": 3.2079999446868896, "metrics/particle/count/prediction_mean": 98.80799865722656, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024216877296566963, "metrics/particle/event/energy_response_mean": 1.0019758939743042, "metrics/particle/event/met_abs_error": 3.8811235427856445, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024097653105854988, "metrics/particle/event/scalar_pt_response_mean": 1.0032457113265991, "metrics/particle/event/vector_pt_closure": 0.026304928585886955, "metrics/particle/matched/delta_eta_abs_mean": 0.008545223623514175, "metrics/particle/matched/delta_phi_abs_mean": 0.00997081771492958, "metrics/particle/matched/delta_r_mean": 0.014667905867099762, "metrics/particle/matched/energy_relative_abs_mean": 0.08545319736003876, "metrics/particle/matched/pid_accuracy": 0.9766952395439148, "metrics/particle/matched/pt_relative_abs_mean": 0.08025691658258438, "metrics/particle/matching/duplicate_fraction": 0.025423042476177216, "metrics/particle/matching/efficiency": 0.8704985976219177, "metrics/particle/matching/f1": 0.86777663230896, "metrics/particle/matching/purity": 0.8650716543197632, "metrics/particle/matching/target_coverage_dr0p05": 0.8478898406028748, "metrics/particle/matching/target_coverage_dr0p10": 0.8980772495269775, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7681685090065002, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9017435312271118}, "step_train_time": 0.8444185256958008, "step_valid_time": 2.037410328979604, "step_total_time": 1004.2390336990356, "elapsed_training_loop_seconds": 1004.2390336990356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011980902933062, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059769581774749336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9527611443779108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540306462358428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533954727030626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04596935376415723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513657561625583, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9507323568575233, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9527611443779108, "step_test_time": 3.5756454950314946, "step_plot_time": 2.7557336480240338, "step_validation_cycle_time": 8.740718006971292} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..800997be055e96eb2b7777839634ea18f8b9702b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 2.1900765895843506, "Classification": 0.015723291784524918, "Regression_pt": 0.10285549610853195, "Regression_eta": 0.006752711720764637, "Regression_sin_phi": 0.0029921643435955048, "Regression_cos_phi": 0.005461377091705799, "Regression_energy": 0.1018788293004036, "Total": 13.971980094909668, "Total_unweighted": 2.425740459933877}, "train_group": {}, "valid": {"Classification_binary": 2.0101475715637207, "Classification": 0.014669091440737247, "Regression_pt": 0.08993910998106003, "Regression_eta": 0.007275431416928768, "Regression_sin_phi": 0.005644860677421093, "Regression_cos_phi": 0.005446342285722494, "Regression_energy": 0.09072395414113998, "Total": 14.36280632019043, "Total_unweighted": 2.2238463615067303, "metrics/particle/class_1/efficiency": 0.9519124031066895, "metrics/particle/class_1/pid_accuracy": 0.9886741638183594, "metrics/particle/class_2/efficiency": 0.5423825979232788, "metrics/particle/class_2/pid_accuracy": 0.8373811841011047, "metrics/particle/class_3/efficiency": 0.8349313735961914, "metrics/particle/class_3/pid_accuracy": 0.9911056160926819, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.5960784554481506, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.7931034564971924, "metrics/particle/count/bias_mean": 0.4440000057220459, "metrics/particle/count/mae": 3.0840001106262207, "metrics/particle/count/prediction_mean": 98.63600158691406, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02557951584458351, "metrics/particle/event/energy_response_mean": 1.004560947418213, "metrics/particle/event/met_abs_error": 4.128810405731201, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02540501393377781, "metrics/particle/event/scalar_pt_response_mean": 1.0070812702178955, "metrics/particle/event/vector_pt_closure": 0.026775648817420006, "metrics/particle/matched/delta_eta_abs_mean": 0.008475450798869133, "metrics/particle/matched/delta_phi_abs_mean": 0.010001049377024174, "metrics/particle/matched/delta_r_mean": 0.014627790078520775, "metrics/particle/matched/energy_relative_abs_mean": 0.08722153306007385, "metrics/particle/matched/pid_accuracy": 0.9767104983329773, "metrics/particle/matched/pt_relative_abs_mean": 0.08131173253059387, "metrics/particle/matching/duplicate_fraction": 0.024980736896395683, "metrics/particle/matching/efficiency": 0.8710689544677734, "metrics/particle/matching/f1": 0.8691039681434631, "metrics/particle/matching/purity": 0.8671479225158691, "metrics/particle/matching/target_coverage_dr0p05": 0.8484601378440857, "metrics/particle/matching/target_coverage_dr0p10": 0.8993808031082153, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7675166726112366, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9007251262664795}, "step_train_time": 0.8706841468811035, "step_valid_time": 2.0116484810132533, "step_total_time": 1089.2836050987244, "elapsed_training_loop_seconds": 1089.2836050987244, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0051342388266382, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06076653308979307, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9527296937416777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524126455906822, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04727030625832224, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9481037924151696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9487350199733688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9484193011647255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.2793341709766537, "step_plot_time": 2.7645814159768634, "step_validation_cycle_time": 8.429075698019005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..0635d1f4489c423eb2c7eb4066d72b65293582c7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 2.0888681411743164, "Classification": 0.01131246518343687, "Regression_pt": 0.08106762915849686, "Regression_eta": 0.00392254488542676, "Regression_sin_phi": 0.0022564188111573458, "Regression_cos_phi": 0.004146702121943235, "Regression_energy": 0.07533305138349533, "Total": 10.567363739013672, "Total_unweighted": 2.266906952718273}, "train_group": {}, "valid": {"Classification_binary": 2.0205814838409424, "Classification": 0.014732267707586288, "Regression_pt": 0.0899648517370224, "Regression_eta": 0.007385672070086002, "Regression_sin_phi": 0.005740780383348465, "Regression_cos_phi": 0.0054457224905490875, "Regression_energy": 0.09071595221757889, "Total": 14.45538330078125, "Total_unweighted": 2.2345667304471135, "metrics/particle/class_1/efficiency": 0.9523401856422424, "metrics/particle/class_1/pid_accuracy": 0.9884995222091675, "metrics/particle/class_2/efficiency": 0.5343642830848694, "metrics/particle/class_2/pid_accuracy": 0.8435155153274536, "metrics/particle/class_3/efficiency": 0.8398194909095764, "metrics/particle/class_3/pid_accuracy": 0.9902619123458862, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.5719844102859497, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7727272510528564, "metrics/particle/count/bias_mean": 0.9120000004768372, "metrics/particle/count/mae": 3.200000047683716, "metrics/particle/count/prediction_mean": 99.10399627685547, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024892985820770264, "metrics/particle/event/energy_response_mean": 0.9990741610527039, "metrics/particle/event/met_abs_error": 4.061331748962402, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024699397385120392, "metrics/particle/event/scalar_pt_response_mean": 1.0015307664871216, "metrics/particle/event/vector_pt_closure": 0.026821138337254524, "metrics/particle/matched/delta_eta_abs_mean": 0.008575026877224445, "metrics/particle/matched/delta_phi_abs_mean": 0.010124566033482552, "metrics/particle/matched/delta_r_mean": 0.014823422767221928, "metrics/particle/matched/energy_relative_abs_mean": 0.08536118268966675, "metrics/particle/matched/pid_accuracy": 0.9761549234390259, "metrics/particle/matched/pt_relative_abs_mean": 0.08046074211597443, "metrics/particle/matching/duplicate_fraction": 0.025347109884023666, "metrics/particle/matching/efficiency": 0.8729835152626038, "metrics/particle/matching/f1": 0.8689481616020203, "metrics/particle/matching/purity": 0.8649499416351318, "metrics/particle/matching/target_coverage_dr0p05": 0.8503747582435608, "metrics/particle/matching/target_coverage_dr0p10": 0.9001140594482422, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7706126570701599, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.903006374835968}, "step_train_time": 0.9201526641845703, "step_valid_time": 2.0476933820173144, "step_total_time": 1173.4647483825684, "elapsed_training_loop_seconds": 1173.4647483825684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013084561528465, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06291805444827148, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9539078156312625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04609218436873747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9505678022712091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9486666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9500998003992016, "step_test_time": 3.6063123180065304, "step_plot_time": 2.7720012749778107, "step_validation_cycle_time": 8.79778050101595} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..355ffb67ffe706e043a16e7c23eff02c491a5f7c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 2.0439741611480713, "Classification": 0.016765393316745758, "Regression_pt": 0.09543602168560028, "Regression_eta": 0.007698907982558012, "Regression_sin_phi": 0.008319995366036892, "Regression_cos_phi": 0.006659574806690216, "Regression_energy": 0.09110014140605927, "Total": 16.449995040893555, "Total_unweighted": 2.2699541957117617}, "train_group": {}, "valid": {"Classification_binary": 2.022202253341675, "Classification": 0.014747410081326962, "Regression_pt": 0.09052853286266327, "Regression_eta": 0.007366378791630268, "Regression_sin_phi": 0.00580932293087244, "Regression_cos_phi": 0.005559357348829508, "Regression_energy": 0.09114576131105423, "Total": 14.544450759887695, "Total_unweighted": 2.2373590166680515, "metrics/particle/class_1/efficiency": 0.9527680277824402, "metrics/particle/class_1/pid_accuracy": 0.9878760576248169, "metrics/particle/class_2/efficiency": 0.5315005779266357, "metrics/particle/class_2/pid_accuracy": 0.8351293206214905, "metrics/particle/class_3/efficiency": 0.8337093591690063, "metrics/particle/class_3/pid_accuracy": 0.9906415343284607, "metrics/particle/class_4/efficiency": 0.8847457766532898, "metrics/particle/class_4/pid_accuracy": 0.6130267977714539, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.7988505959510803, "metrics/particle/count/bias_mean": 0.8600000143051147, "metrics/particle/count/mae": 3.180000066757202, "metrics/particle/count/prediction_mean": 99.052001953125, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025569403544068336, "metrics/particle/event/energy_response_mean": 1.0085530281066895, "metrics/particle/event/met_abs_error": 4.23492431640625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0264348853379488, "metrics/particle/event/scalar_pt_response_mean": 1.0102661848068237, "metrics/particle/event/vector_pt_closure": 0.02720319852232933, "metrics/particle/matched/delta_eta_abs_mean": 0.008629889227449894, "metrics/particle/matched/delta_phi_abs_mean": 0.009967739693820477, "metrics/particle/matched/delta_r_mean": 0.014738563448190689, "metrics/particle/matched/energy_relative_abs_mean": 0.08596166223287582, "metrics/particle/matched/pid_accuracy": 0.9762718081474304, "metrics/particle/matched/pt_relative_abs_mean": 0.08070311695337296, "metrics/particle/matching/duplicate_fraction": 0.0259257759898901, "metrics/particle/matching/efficiency": 0.8704171180725098, "metrics/particle/matching/f1": 0.8666220307350159, "metrics/particle/matching/purity": 0.8628599047660828, "metrics/particle/matching/target_coverage_dr0p05": 0.8483379483222961, "metrics/particle/matching/target_coverage_dr0p10": 0.8992586135864258, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7671093344688416, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9008066058158875}, "step_train_time": 0.8701660633087158, "step_valid_time": 2.0419321080553345, "step_total_time": 1257.4151656627655, "elapsed_training_loop_seconds": 1257.4151656627655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054304367800104, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059745851019832874, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1500, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9527611443779108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9546666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537129537129537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04533333333333334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513333333333334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503829503829504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9527611443779108, "step_test_time": 3.3488445010152645, "step_plot_time": 2.8911146199679933, "step_validation_cycle_time": 8.657234424958006} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab929359cca2d935777593d924c7e2165c45bdb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 1.9383305311203003, "Classification": 0.014783725142478943, "Regression_pt": 0.07165663689374924, "Regression_eta": 0.002390847774222493, "Regression_sin_phi": 0.0035764744970947504, "Regression_cos_phi": 0.0025825323536992073, "Regression_energy": 0.07146129757165909, "Total": 9.98647689819336, "Total_unweighted": 2.104782045353204}, "train_group": {}, "valid": {"Classification_binary": 2.0271682739257812, "Classification": 0.014780950732529163, "Regression_pt": 0.09126609563827515, "Regression_eta": 0.007427085656672716, "Regression_sin_phi": 0.005795732606202364, "Regression_cos_phi": 0.005510049406439066, "Regression_energy": 0.09214966744184494, "Total": 14.58458423614502, "Total_unweighted": 2.2440978554077446, "metrics/particle/class_1/efficiency": 0.9524257779121399, "metrics/particle/class_1/pid_accuracy": 0.9877818822860718, "metrics/particle/class_2/efficiency": 0.5320733189582825, "metrics/particle/class_2/pid_accuracy": 0.8406888842582703, "metrics/particle/class_3/efficiency": 0.8308892846107483, "metrics/particle/class_3/pid_accuracy": 0.991062343120575, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.6070038676261902, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.7988505959510803, "metrics/particle/count/bias_mean": 0.3240000009536743, "metrics/particle/count/mae": 3.115999937057495, "metrics/particle/count/prediction_mean": 98.51599884033203, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024755435064435005, "metrics/particle/event/energy_response_mean": 1.001093864440918, "metrics/particle/event/met_abs_error": 3.9704935550689697, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02487506903707981, "metrics/particle/event/scalar_pt_response_mean": 1.0040992498397827, "metrics/particle/event/vector_pt_closure": 0.026953067630529404, "metrics/particle/matched/delta_eta_abs_mean": 0.008654782548546791, "metrics/particle/matched/delta_phi_abs_mean": 0.010096569545567036, "metrics/particle/matched/delta_r_mean": 0.014854107052087784, "metrics/particle/matched/energy_relative_abs_mean": 0.08574897795915604, "metrics/particle/matched/pid_accuracy": 0.9766057133674622, "metrics/particle/matched/pt_relative_abs_mean": 0.08040622621774673, "metrics/particle/matching/duplicate_fraction": 0.024523934349417686, "metrics/particle/matching/efficiency": 0.8689098954200745, "metrics/particle/matching/f1": 0.8674787282943726, "metrics/particle/matching/purity": 0.8660522103309631, "metrics/particle/matching/target_coverage_dr0p05": 0.8454456329345703, "metrics/particle/matching/target_coverage_dr0p10": 0.8969773650169373, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7671500444412231, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8991771340370178}, "step_train_time": 0.8553564548492432, "step_valid_time": 2.031714915006887, "step_total_time": 1342.2949452400208, "elapsed_training_loop_seconds": 1342.2949452400208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030734137394648, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061820786720299115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573048699132756, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9560293137908061, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04269513008672448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533022014676451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9520319786808794, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547571523619428, "step_test_time": 3.5572530510253273, "step_plot_time": 2.7722347999806516, "step_validation_cycle_time": 8.736319774994627} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a4aceb25103fbb6c4f9c24c41dd11a850c4d7a7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 1.9702876806259155, "Classification": 0.010717413388192654, "Regression_pt": 0.0762958824634552, "Regression_eta": 0.006331094074994326, "Regression_sin_phi": 0.006824387703090906, "Regression_cos_phi": 0.0050443257205188274, "Regression_energy": 0.07723067700862885, "Total": 13.24169921875, "Total_unweighted": 2.1527314609847963}, "train_group": {}, "valid": {"Classification_binary": 2.0302202701568604, "Classification": 0.014856737107038498, "Regression_pt": 0.09136486053466797, "Regression_eta": 0.0074934642761945724, "Regression_sin_phi": 0.005944796372205019, "Regression_cos_phi": 0.00564758712425828, "Regression_energy": 0.0922296941280365, "Total": 14.735525131225586, "Total_unweighted": 2.247757409699261, "metrics/particle/class_1/efficiency": 0.9535381197929382, "metrics/particle/class_1/pid_accuracy": 0.9884242415428162, "metrics/particle/class_2/efficiency": 0.5349370241165161, "metrics/particle/class_2/pid_accuracy": 0.8415417671203613, "metrics/particle/class_3/efficiency": 0.8339913487434387, "metrics/particle/class_3/pid_accuracy": 0.9909828901290894, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.60546875, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7853107452392578, "metrics/particle/count/bias_mean": 0.41600000858306885, "metrics/particle/count/mae": 3.2320001125335693, "metrics/particle/count/prediction_mean": 98.60800170898438, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.0244746096432209, "metrics/particle/event/energy_response_mean": 0.9993067383766174, "metrics/particle/event/met_abs_error": 4.085625648498535, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024537939578294754, "metrics/particle/event/scalar_pt_response_mean": 1.000382900238037, "metrics/particle/event/vector_pt_closure": 0.02678431198000908, "metrics/particle/matched/delta_eta_abs_mean": 0.00865003652870655, "metrics/particle/matched/delta_phi_abs_mean": 0.009967238642275333, "metrics/particle/matched/delta_r_mean": 0.014735203236341476, "metrics/particle/matched/energy_relative_abs_mean": 0.08544587343931198, "metrics/particle/matched/pid_accuracy": 0.9768040180206299, "metrics/particle/matched/pt_relative_abs_mean": 0.08030525594949722, "metrics/particle/matching/duplicate_fraction": 0.024825571104884148, "metrics/particle/matching/efficiency": 0.8710689544677734, "metrics/particle/matching/f1": 0.8692276477813721, "metrics/particle/matching/purity": 0.8673941493034363, "metrics/particle/matching/target_coverage_dr0p05": 0.8468307256698608, "metrics/particle/matching/target_coverage_dr0p10": 0.8982402086257935, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7681277394294739, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9005621671676636}, "step_train_time": 0.8298182487487793, "step_valid_time": 2.0344802570180036, "step_total_time": 1425.7881858348846, "elapsed_training_loop_seconds": 1425.7881858348846, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014412395389638, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06491845920047323, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9567531603459747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580279813457695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573901464713716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04197201865423051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9527611443779108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540306462358428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533954727030626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9567531603459747, "step_test_time": 3.237924838962499, "step_plot_time": 2.762138487014454, "step_validation_cycle_time": 8.408237075957004} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..592b2b69cb476ff87e28d0ab360d2de8ff0ff037 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 1.9377684593200684, "Classification": 0.015127938240766525, "Regression_pt": 0.09429916739463806, "Regression_eta": 0.003697450738400221, "Regression_sin_phi": 0.005676500499248505, "Regression_cos_phi": 0.0053912135772407055, "Regression_energy": 0.09164192527532578, "Total": 13.26380443572998, "Total_unweighted": 2.153602655045688}, "train_group": {}, "valid": {"Classification_binary": 2.0358564853668213, "Classification": 0.014998148195445538, "Regression_pt": 0.09147538244724274, "Regression_eta": 0.007433839607983828, "Regression_sin_phi": 0.005819183774292469, "Regression_cos_phi": 0.005580573808401823, "Regression_energy": 0.09245636314153671, "Total": 14.671377182006836, "Total_unweighted": 2.2536199763417244, "metrics/particle/class_1/efficiency": 0.9512278437614441, "metrics/particle/class_1/pid_accuracy": 0.9880363345146179, "metrics/particle/class_2/efficiency": 0.5297823548316956, "metrics/particle/class_2/pid_accuracy": 0.846486508846283, "metrics/particle/class_3/efficiency": 0.8369994163513184, "metrics/particle/class_3/pid_accuracy": 0.991127610206604, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.6171875, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7897727489471436, "metrics/particle/count/bias_mean": 0.6800000071525574, "metrics/particle/count/mae": 3.247999906539917, "metrics/particle/count/prediction_mean": 98.87200164794922, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.0253602284938097, "metrics/particle/event/energy_response_mean": 1.0016621351242065, "metrics/particle/event/met_abs_error": 4.15437126159668, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02489587850868702, "metrics/particle/event/scalar_pt_response_mean": 1.0041699409484863, "metrics/particle/event/vector_pt_closure": 0.026936586946249008, "metrics/particle/matched/delta_eta_abs_mean": 0.008719762787222862, "metrics/particle/matched/delta_phi_abs_mean": 0.010081273503601551, "metrics/particle/matched/delta_r_mean": 0.01489270944148302, "metrics/particle/matched/energy_relative_abs_mean": 0.08514881134033203, "metrics/particle/matched/pid_accuracy": 0.9771260023117065, "metrics/particle/matched/pt_relative_abs_mean": 0.08034222573041916, "metrics/particle/matching/duplicate_fraction": 0.025082934647798538, "metrics/particle/matching/efficiency": 0.8708652257919312, "metrics/particle/matching/f1": 0.8678601980209351, "metrics/particle/matching/purity": 0.8648757934570312, "metrics/particle/matching/target_coverage_dr0p05": 0.8462196588516235, "metrics/particle/matching/target_coverage_dr0p10": 0.898851215839386, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7695127725601196, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9011324644088745}, "step_train_time": 0.8368463516235352, "step_valid_time": 2.051438231021166, "step_total_time": 1510.5793936252594, "elapsed_training_loop_seconds": 1510.5793936252594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024482145535298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05998225597260709, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9527611443779108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540306462358428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533954727030626, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04596935376415723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506995336442372, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9500665778961385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9527611443779108, "step_test_time": 3.289226458990015, "step_plot_time": 2.8402069919975474, "step_validation_cycle_time": 8.547130474995356} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..15241910998894872dbe5a8eea9e366ea8426393 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 2.0556063652038574, "Classification": 0.01472581084817648, "Regression_pt": 0.09254030883312225, "Regression_eta": 0.002985980361700058, "Regression_sin_phi": 0.0034073691349476576, "Regression_cos_phi": 0.0028203404508531094, "Regression_energy": 0.09276782721281052, "Total": 11.181188583374023, "Total_unweighted": 2.2648540020454675}, "train_group": {}, "valid": {"Classification_binary": 2.0405173301696777, "Classification": 0.015146497637033463, "Regression_pt": 0.09178569912910461, "Regression_eta": 0.007605796679854393, "Regression_sin_phi": 0.005952014587819576, "Regression_cos_phi": 0.005692162550985813, "Regression_energy": 0.09279607236385345, "Total": 14.86136245727539, "Total_unweighted": 2.259495573118329, "metrics/particle/class_1/efficiency": 0.9509711861610413, "metrics/particle/class_1/pid_accuracy": 0.9874932765960693, "metrics/particle/class_2/efficiency": 0.5297823548316956, "metrics/particle/class_2/pid_accuracy": 0.838918924331665, "metrics/particle/class_3/efficiency": 0.8332393169403076, "metrics/particle/class_3/pid_accuracy": 0.9904106259346008, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.6177605986595154, "metrics/particle/class_5/efficiency": 0.9835164546966553, "metrics/particle/class_5/pid_accuracy": 0.7765362858772278, "metrics/particle/count/bias_mean": 0.5479999780654907, "metrics/particle/count/mae": 3.131999969482422, "metrics/particle/count/prediction_mean": 98.73999786376953, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02500063367187977, "metrics/particle/event/energy_response_mean": 1.0016874074935913, "metrics/particle/event/met_abs_error": 4.212560653686523, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025394149124622345, "metrics/particle/event/scalar_pt_response_mean": 1.0033904314041138, "metrics/particle/event/vector_pt_closure": 0.02785760723054409, "metrics/particle/matched/delta_eta_abs_mean": 0.008763931691646576, "metrics/particle/matched/delta_phi_abs_mean": 0.010031326673924923, "metrics/particle/matched/delta_r_mean": 0.01489320769906044, "metrics/particle/matched/energy_relative_abs_mean": 0.08592018485069275, "metrics/particle/matched/pid_accuracy": 0.9760085940361023, "metrics/particle/matched/pt_relative_abs_mean": 0.08059577643871307, "metrics/particle/matching/duplicate_fraction": 0.024670852348208427, "metrics/particle/matching/efficiency": 0.8693580031394958, "metrics/particle/matching/f1": 0.8669388294219971, "metrics/particle/matching/purity": 0.8645331263542175, "metrics/particle/matching/target_coverage_dr0p05": 0.8471973538398743, "metrics/particle/matching/target_coverage_dr0p10": 0.8977921009063721, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.766905665397644, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8999918699264526}, "step_train_time": 0.8161852359771729, "step_valid_time": 2.028473070997279, "step_total_time": 1594.6659519672394, "elapsed_training_loop_seconds": 1594.6659519672394, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002484315417949, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06358511933981603, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545758183032732, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9526666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04542418169672679, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461077844311377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9498997995991983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507651363938789, "step_test_time": 3.369639308017213, "step_plot_time": 2.751200696977321, "step_validation_cycle_time": 8.521087414992508} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..db1aacb2a9f976999a6438fb51177cef4fafd686 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 2.1934614181518555, "Classification": 0.015904126688838005, "Regression_pt": 0.10535817593336105, "Regression_eta": 0.005807298235595226, "Regression_sin_phi": 0.0043975128792226315, "Regression_cos_phi": 0.006000075489282608, "Regression_energy": 0.1060173511505127, "Total": 14.581064224243164, "Total_unweighted": 2.4369459585286677}, "train_group": {}, "valid": {"Classification_binary": 1.9778952598571777, "Classification": 0.014276297762989998, "Regression_pt": 0.08663250505924225, "Regression_eta": 0.006938575766980648, "Regression_sin_phi": 0.005525484681129456, "Regression_cos_phi": 0.005299548618495464, "Regression_energy": 0.08713437616825104, "Total": 13.9227933883667, "Total_unweighted": 2.1837020479142666, "metrics/particle/class_1/efficiency": 0.9509711861610413, "metrics/particle/class_1/pid_accuracy": 0.988213062286377, "metrics/particle/class_2/efficiency": 0.5481099486351013, "metrics/particle/class_2/pid_accuracy": 0.8443050980567932, "metrics/particle/class_3/efficiency": 0.8340853452682495, "metrics/particle/class_3/pid_accuracy": 0.9908711910247803, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.6216216087341309, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7683615684509277, "metrics/particle/count/bias_mean": 0.6240000128746033, "metrics/particle/count/mae": 3.2079999446868896, "metrics/particle/count/prediction_mean": 98.81600189208984, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025071896612644196, "metrics/particle/event/energy_response_mean": 1.000948190689087, "metrics/particle/event/met_abs_error": 4.060028553009033, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02411719597876072, "metrics/particle/event/scalar_pt_response_mean": 1.002490520477295, "metrics/particle/event/vector_pt_closure": 0.026040874421596527, "metrics/particle/matched/delta_eta_abs_mean": 0.008466125465929508, "metrics/particle/matched/delta_phi_abs_mean": 0.009560824371874332, "metrics/particle/matched/delta_r_mean": 0.014268847182393074, "metrics/particle/matched/energy_relative_abs_mean": 0.08654362708330154, "metrics/particle/matched/pid_accuracy": 0.9766136407852173, "metrics/particle/matched/pt_relative_abs_mean": 0.08124534785747528, "metrics/particle/matching/duplicate_fraction": 0.024894753471016884, "metrics/particle/matching/efficiency": 0.8709467053413391, "metrics/particle/matching/f1": 0.8681880831718445, "metrics/particle/matching/purity": 0.8654468655586243, "metrics/particle/matching/target_coverage_dr0p05": 0.8505784869194031, "metrics/particle/matching/target_coverage_dr0p10": 0.898769736289978, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7654391527175903, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.900439977645874}, "step_train_time": 0.9165647029876709, "step_valid_time": 1.9882037279894575, "step_total_time": 162.15824222564697, "elapsed_training_loop_seconds": 162.15824222564697, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026141226142011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05900723702356059, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9534264803725881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540612516644474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537437603993344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045938748335552594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513981358189081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95108153078203, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9534264803725881, "step_test_time": 3.4693979090079665, "step_plot_time": 2.722905214992352, "step_validation_cycle_time": 8.58798728697002} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad98b2b06ab96597b0c7b47f4ae8de7649e6c04d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 1.7791595458984375, "Classification": 0.013369651511311531, "Regression_pt": 0.08733627945184708, "Regression_eta": 0.006749200634658337, "Regression_sin_phi": 0.0056098448112607, "Regression_cos_phi": 0.005593434441834688, "Regression_energy": 0.08062893897294998, "Total": 13.573700904846191, "Total_unweighted": 1.9784468957222998}, "train_group": {}, "valid": {"Classification_binary": 2.049863815307617, "Classification": 0.015068499371409416, "Regression_pt": 0.09274450689554214, "Regression_eta": 0.007424953859299421, "Regression_sin_phi": 0.005906221456825733, "Regression_cos_phi": 0.0056197308003902435, "Regression_energy": 0.09384254366159439, "Total": 14.79506778717041, "Total_unweighted": 2.2704702713526785, "metrics/particle/class_1/efficiency": 0.9520835280418396, "metrics/particle/class_1/pid_accuracy": 0.9873281121253967, "metrics/particle/class_2/efficiency": 0.5337915420532227, "metrics/particle/class_2/pid_accuracy": 0.8315450549125671, "metrics/particle/class_3/efficiency": 0.8328633308410645, "metrics/particle/class_3/pid_accuracy": 0.9919864535331726, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.61328125, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7897727489471436, "metrics/particle/count/bias_mean": 0.20800000429153442, "metrics/particle/count/mae": 3.119999885559082, "metrics/particle/count/prediction_mean": 98.4000015258789, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024422571063041687, "metrics/particle/event/energy_response_mean": 0.997981607913971, "metrics/particle/event/met_abs_error": 3.9743754863739014, "metrics/particle/event/scalar_pt_relative_abs_error": 0.023987799882888794, "metrics/particle/event/scalar_pt_response_mean": 0.9999958872795105, "metrics/particle/event/vector_pt_closure": 0.02725330740213394, "metrics/particle/matched/delta_eta_abs_mean": 0.009044473059475422, "metrics/particle/matched/delta_phi_abs_mean": 0.009997686371207237, "metrics/particle/matched/delta_r_mean": 0.015064168721437454, "metrics/particle/matched/energy_relative_abs_mean": 0.0861746072769165, "metrics/particle/matched/pid_accuracy": 0.9763476848602295, "metrics/particle/matched/pt_relative_abs_mean": 0.08055282384157181, "metrics/particle/matching/duplicate_fraction": 0.02386178821325302, "metrics/particle/matching/efficiency": 0.8697653412818909, "metrics/particle/matching/f1": 0.8688451051712036, "metrics/particle/matching/purity": 0.867926836013794, "metrics/particle/matching/target_coverage_dr0p05": 0.8473195433616638, "metrics/particle/matching/target_coverage_dr0p10": 0.8980772495269775, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7678425908088684, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8992993235588074}, "step_train_time": 0.8732075691223145, "step_valid_time": 2.026112766994629, "step_total_time": 1679.3522913455963, "elapsed_training_loop_seconds": 1679.3522913455963, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019288180040797, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061654243221102134, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9527611443779108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9565798263193053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04342017368069472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539078156312625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9527611443779108, "step_test_time": 3.5721749319927767, "step_plot_time": 2.7852273829630576, "step_validation_cycle_time": 8.745177308970597} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..cfd1c5b84b9ce49aa27278cc5eb7cd71ba281eb7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 1.7134191989898682, "Classification": 0.013193086721003056, "Regression_pt": 0.07162441313266754, "Regression_eta": 0.0035046518314629793, "Regression_sin_phi": 0.002648951020091772, "Regression_cos_phi": 0.004227466881275177, "Regression_energy": 0.07161903381347656, "Total": 10.215629577636719, "Total_unweighted": 1.8802368023898453}, "train_group": {}, "valid": {"Classification_binary": 2.049853801727295, "Classification": 0.01528982724994421, "Regression_pt": 0.09259867668151855, "Regression_eta": 0.007523860316723585, "Regression_sin_phi": 0.006047836970537901, "Regression_cos_phi": 0.005768441129475832, "Regression_energy": 0.09367720037698746, "Total": 14.964238166809082, "Total_unweighted": 2.2707596444524825, "metrics/particle/class_1/efficiency": 0.9530247449874878, "metrics/particle/class_1/pid_accuracy": 0.9880589246749878, "metrics/particle/class_2/efficiency": 0.5280641317367554, "metrics/particle/class_2/pid_accuracy": 0.8427332043647766, "metrics/particle/class_3/efficiency": 0.8334273099899292, "metrics/particle/class_3/pid_accuracy": 0.9913151264190674, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.6070038676261902, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7865168452262878, "metrics/particle/count/bias_mean": 0.492000013589859, "metrics/particle/count/mae": 3.2039999961853027, "metrics/particle/count/prediction_mean": 98.68399810791016, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025721482932567596, "metrics/particle/event/energy_response_mean": 1.0014828443527222, "metrics/particle/event/met_abs_error": 4.191562652587891, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02544543147087097, "metrics/particle/event/scalar_pt_response_mean": 1.0024311542510986, "metrics/particle/event/vector_pt_closure": 0.02770426496863365, "metrics/particle/matched/delta_eta_abs_mean": 0.008626722730696201, "metrics/particle/matched/delta_phi_abs_mean": 0.010368457064032555, "metrics/particle/matched/delta_r_mean": 0.015061568468809128, "metrics/particle/matched/energy_relative_abs_mean": 0.08647234737873077, "metrics/particle/matched/pid_accuracy": 0.9768737554550171, "metrics/particle/matched/pt_relative_abs_mean": 0.08051155507564545, "metrics/particle/matching/duplicate_fraction": 0.02419845201075077, "metrics/particle/matching/efficiency": 0.8701727390289307, "metrics/particle/matching/f1": 0.8679981231689453, "metrics/particle/matching/purity": 0.8658343553543091, "metrics/particle/matching/target_coverage_dr0p05": 0.8436939716339111, "metrics/particle/matching/target_coverage_dr0p10": 0.8965699672698975, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7678425908088684, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9011732339859009}, "step_train_time": 0.8344902992248535, "step_valid_time": 2.087398659030441, "step_total_time": 1763.6428844928741, "elapsed_training_loop_seconds": 1763.6428844928741, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016985845490036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06282231282367434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9554224883566201, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9598930481283422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9576525508502834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040106951871657755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534264803725881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9578877005347594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556518839613204, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9554224883566201, "step_test_time": 3.491639414045494, "step_plot_time": 2.7705538329901174, "step_validation_cycle_time": 8.71422598301433} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..4beeb5766718459550f7cb850f13d52f19e4b35a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 2.3247978687286377, "Classification": 0.012497234158217907, "Regression_pt": 0.11554349958896637, "Regression_eta": 0.008905564434826374, "Regression_sin_phi": 0.0034798895940184593, "Regression_cos_phi": 0.003044006647542119, "Regression_energy": 0.13004335761070251, "Total": 14.464709281921387, "Total_unweighted": 2.5983114207629114}, "train_group": {}, "valid": {"Classification_binary": 2.0536253452301025, "Classification": 0.015376200899481773, "Regression_pt": 0.09344398230314255, "Regression_eta": 0.00755700608715415, "Regression_sin_phi": 0.006070329342037439, "Regression_cos_phi": 0.005771944765001535, "Regression_energy": 0.0948328971862793, "Total": 15.04116153717041, "Total_unweighted": 2.2766777058131993, "metrics/particle/class_1/efficiency": 0.9524257779121399, "metrics/particle/class_1/pid_accuracy": 0.987422525882721, "metrics/particle/class_2/efficiency": 0.5217640399932861, "metrics/particle/class_2/pid_accuracy": 0.8342480659484863, "metrics/particle/class_3/efficiency": 0.8359653949737549, "metrics/particle/class_3/pid_accuracy": 0.9915664196014404, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.6196078658103943, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7808988690376282, "metrics/particle/count/bias_mean": 0.5799999833106995, "metrics/particle/count/mae": 3.0920000076293945, "metrics/particle/count/prediction_mean": 98.77200317382812, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02452484518289566, "metrics/particle/event/energy_response_mean": 1.0038416385650635, "metrics/particle/event/met_abs_error": 4.209734916687012, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025138413533568382, "metrics/particle/event/scalar_pt_response_mean": 1.005183219909668, "metrics/particle/event/vector_pt_closure": 0.028227543458342552, "metrics/particle/matched/delta_eta_abs_mean": 0.008608284406363964, "metrics/particle/matched/delta_phi_abs_mean": 0.010029100812971592, "metrics/particle/matched/delta_r_mean": 0.014752114191651344, "metrics/particle/matched/energy_relative_abs_mean": 0.0867384523153305, "metrics/particle/matched/pid_accuracy": 0.9765069484710693, "metrics/particle/matched/pt_relative_abs_mean": 0.08071771264076233, "metrics/particle/matching/duplicate_fraction": 0.02535131387412548, "metrics/particle/matching/efficiency": 0.8704578876495361, "metrics/particle/matching/f1": 0.8678946495056152, "metrics/particle/matching/purity": 0.8653464317321777, "metrics/particle/matching/target_coverage_dr0p05": 0.84858238697052, "metrics/particle/matching/target_coverage_dr0p10": 0.898851215839386, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7671500444412231, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8993808031082153}, "step_train_time": 0.8934175968170166, "step_valid_time": 2.012880617985502, "step_total_time": 1847.9045453071594, "elapsed_training_loop_seconds": 1847.9045453071594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003704252127618, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06389866686265899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9567531603459747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580279813457695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573901464713716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04197201865423051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9540918163672655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.955363091272485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9547270306258322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9567531603459747, "step_test_time": 3.2555310890311375, "step_plot_time": 2.800162066007033, "step_validation_cycle_time": 8.442385111993644} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..cd9613a41fb64ef06f3b4713d539aa04564a89eb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 2.1547393798828125, "Classification": 0.01214162539690733, "Regression_pt": 0.0856224000453949, "Regression_eta": 0.005186939612030983, "Regression_sin_phi": 0.003928087186068296, "Regression_cos_phi": 0.004427072126418352, "Regression_energy": 0.08661823719739914, "Total": 12.243938446044922, "Total_unweighted": 2.3526637414470315}, "train_group": {}, "valid": {"Classification_binary": 2.0602095127105713, "Classification": 0.015521585009992123, "Regression_pt": 0.09323137998580933, "Regression_eta": 0.007743721827864647, "Regression_sin_phi": 0.006199279334396124, "Regression_cos_phi": 0.005815271288156509, "Regression_energy": 0.09443285316228867, "Total": 15.183308601379395, "Total_unweighted": 2.2831536033190787, "metrics/particle/class_1/efficiency": 0.9522546529769897, "metrics/particle/class_1/pid_accuracy": 0.9883188009262085, "metrics/particle/class_2/efficiency": 0.5223367810249329, "metrics/particle/class_2/pid_accuracy": 0.8410087823867798, "metrics/particle/class_3/efficiency": 0.8344613909721375, "metrics/particle/class_3/pid_accuracy": 0.9908753037452698, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.609375, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7796609997749329, "metrics/particle/count/bias_mean": 0.36000001430511475, "metrics/particle/count/mae": 3.1519999504089355, "metrics/particle/count/prediction_mean": 98.552001953125, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02588082291185856, "metrics/particle/event/energy_response_mean": 1.0006258487701416, "metrics/particle/event/met_abs_error": 4.307280540466309, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02586798369884491, "metrics/particle/event/scalar_pt_response_mean": 1.002445936203003, "metrics/particle/event/vector_pt_closure": 0.02835429459810257, "metrics/particle/matched/delta_eta_abs_mean": 0.008553499355912209, "metrics/particle/matched/delta_phi_abs_mean": 0.010026255622506142, "metrics/particle/matched/delta_r_mean": 0.014728412963449955, "metrics/particle/matched/energy_relative_abs_mean": 0.08600638061761856, "metrics/particle/matched/pid_accuracy": 0.9768160581588745, "metrics/particle/matched/pt_relative_abs_mean": 0.08058244735002518, "metrics/particle/matching/duplicate_fraction": 0.02479909174144268, "metrics/particle/matching/efficiency": 0.8697653412818909, "metrics/particle/matching/f1": 0.8681738972663879, "metrics/particle/matching/purity": 0.8665881752967834, "metrics/particle/matching/target_coverage_dr0p05": 0.8467084765434265, "metrics/particle/matching/target_coverage_dr0p10": 0.8973439931869507, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.766987144947052, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9006028771400452}, "step_train_time": 0.8437855243682861, "step_valid_time": 2.018002303026151, "step_total_time": 1932.1935505867004, "elapsed_training_loop_seconds": 1932.1935505867004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029111617657147, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06461201726882015, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579439252336449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9563478840386538, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04205607476635514, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539385847797063, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9523492169276908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547571523619428, "step_test_time": 3.3390077670337632, "step_plot_time": 2.775543424009811, "step_validation_cycle_time": 8.515067989996169} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..43b62d01498559ad4d48b71556dc67add5a7965f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 1.7863761186599731, "Classification": 0.012589427642524242, "Regression_pt": 0.07236114889383316, "Regression_eta": 0.0016529887216165662, "Regression_sin_phi": 0.0024998020380735397, "Regression_cos_phi": 0.002485039411112666, "Regression_energy": 0.07423260807991028, "Total": 8.917776107788086, "Total_unweighted": 1.9521971334470436}, "train_group": {}, "valid": {"Classification_binary": 2.0616283416748047, "Classification": 0.01559791062027216, "Regression_pt": 0.09445732086896896, "Regression_eta": 0.0076048653572797775, "Regression_sin_phi": 0.0063016777858138084, "Regression_cos_phi": 0.005789461079984903, "Regression_energy": 0.09578491002321243, "Total": 15.230855941772461, "Total_unweighted": 2.2871644874103367, "metrics/particle/class_1/efficiency": 0.9513989686965942, "metrics/particle/class_1/pid_accuracy": 0.9877687096595764, "metrics/particle/class_2/efficiency": 0.5223367810249329, "metrics/particle/class_2/pid_accuracy": 0.8344298005104065, "metrics/particle/class_3/efficiency": 0.8285391926765442, "metrics/particle/class_3/pid_accuracy": 0.9917176961898804, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.6031128168106079, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7865168452262878, "metrics/particle/count/bias_mean": 0.4399999976158142, "metrics/particle/count/mae": 3.2160000801086426, "metrics/particle/count/prediction_mean": 98.63200378417969, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025210769847035408, "metrics/particle/event/energy_response_mean": 0.9975325465202332, "metrics/particle/event/met_abs_error": 4.324841499328613, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024984873831272125, "metrics/particle/event/scalar_pt_response_mean": 0.9991652965545654, "metrics/particle/event/vector_pt_closure": 0.0282409880310297, "metrics/particle/matched/delta_eta_abs_mean": 0.008993757888674736, "metrics/particle/matched/delta_phi_abs_mean": 0.010140511207282543, "metrics/particle/matched/delta_r_mean": 0.015166401863098145, "metrics/particle/matched/energy_relative_abs_mean": 0.08541349321603775, "metrics/particle/matched/pid_accuracy": 0.9765037298202515, "metrics/particle/matched/pt_relative_abs_mean": 0.08040879666805267, "metrics/particle/matching/duplicate_fraction": 0.025062859058380127, "metrics/particle/matching/efficiency": 0.8668730854988098, "metrics/particle/matching/f1": 0.8649351596832275, "metrics/particle/matching/purity": 0.8630059361457825, "metrics/particle/matching/target_coverage_dr0p05": 0.8401091694831848, "metrics/particle/matching/target_coverage_dr0p10": 0.8951034545898438, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7653576731681824, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8972217440605164}, "step_train_time": 0.8222455978393555, "step_valid_time": 2.0228165700100362, "step_total_time": 2016.9950153827667, "elapsed_training_loop_seconds": 2016.9950153827667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021438834969096, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06333945680917785, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9546364242828552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533644237175216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04536357571714476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.95263509006004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9513657561625583, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.5228708049980924, "step_plot_time": 2.775496725051198, "step_validation_cycle_time": 8.713884496013634} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..56dc2fdd2fa36548a5945ea419796090035fc1f9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 1.8028932809829712, "Classification": 0.009998675435781479, "Regression_pt": 0.07182884961366653, "Regression_eta": 0.0039297747425735, "Regression_sin_phi": 0.003877337323501706, "Regression_cos_phi": 0.004223071038722992, "Regression_energy": 0.06926871091127396, "Total": 10.461325645446777, "Total_unweighted": 1.9660197000484914}, "train_group": {}, "valid": {"Classification_binary": 2.0682926177978516, "Classification": 0.015670085325837135, "Regression_pt": 0.09432125836610794, "Regression_eta": 0.00772115308791399, "Regression_sin_phi": 0.006354242563247681, "Regression_cos_phi": 0.005860874895006418, "Regression_energy": 0.09582699835300446, "Total": 15.332221031188965, "Total_unweighted": 2.294047230388969, "metrics/particle/class_1/efficiency": 0.9513989686965942, "metrics/particle/class_1/pid_accuracy": 0.9875888228416443, "metrics/particle/class_2/efficiency": 0.5229095220565796, "metrics/particle/class_2/pid_accuracy": 0.8389923572540283, "metrics/particle/class_3/efficiency": 0.8291972279548645, "metrics/particle/class_3/pid_accuracy": 0.9912708401679993, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.5764706134796143, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7897727489471436, "metrics/particle/count/bias_mean": 0.37599998712539673, "metrics/particle/count/mae": 3.1040000915527344, "metrics/particle/count/prediction_mean": 98.56800079345703, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025376025587320328, "metrics/particle/event/energy_response_mean": 1.0012956857681274, "metrics/particle/event/met_abs_error": 4.077924728393555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025100480765104294, "metrics/particle/event/scalar_pt_response_mean": 1.003208041191101, "metrics/particle/event/vector_pt_closure": 0.027486737817525864, "metrics/particle/matched/delta_eta_abs_mean": 0.008607328869402409, "metrics/particle/matched/delta_phi_abs_mean": 0.010220005176961422, "metrics/particle/matched/delta_r_mean": 0.014923975802958012, "metrics/particle/matched/energy_relative_abs_mean": 0.08662768453359604, "metrics/particle/matched/pid_accuracy": 0.976179301738739, "metrics/particle/matched/pt_relative_abs_mean": 0.08081255108118057, "metrics/particle/matching/duplicate_fraction": 0.025079132989048958, "metrics/particle/matching/efficiency": 0.867035984992981, "metrics/particle/matching/f1": 0.8653791546821594, "metrics/particle/matching/purity": 0.8637285828590393, "metrics/particle/matching/target_coverage_dr0p05": 0.8436124920845032, "metrics/particle/matching/target_coverage_dr0p10": 0.8959181904792786, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7645429372787476, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8970995545387268}, "step_train_time": 0.8676414489746094, "step_valid_time": 2.0051367409760132, "step_total_time": 2100.4661769866943, "elapsed_training_loop_seconds": 2100.4661769866943, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032206857627046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06254835232231981, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553928095872171, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9550748752079867, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04460719041278296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513981358189081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95108153078203, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547571523619428, "step_test_time": 3.4848925080150366, "step_plot_time": 2.767849233991001, "step_validation_cycle_time": 8.635547503014095} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..bb7280d16ed935b7f55e237101062bd9973992a7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 1.8516637086868286, "Classification": 0.012159857898950577, "Regression_pt": 0.07013950496912003, "Regression_eta": 0.001827359083108604, "Regression_sin_phi": 0.0014390262076631188, "Regression_cos_phi": 0.0023598247207701206, "Regression_energy": 0.0692538395524025, "Total": 8.375877380371094, "Total_unweighted": 2.0088431211188436}, "train_group": {}, "valid": {"Classification_binary": 2.0683186054229736, "Classification": 0.015676217153668404, "Regression_pt": 0.09437192976474762, "Regression_eta": 0.007816210389137268, "Regression_sin_phi": 0.006244522053748369, "Regression_cos_phi": 0.005760750267654657, "Regression_energy": 0.09580279141664505, "Total": 15.282755851745605, "Total_unweighted": 2.293991026468575, "metrics/particle/class_1/efficiency": 0.9528536200523376, "metrics/particle/class_1/pid_accuracy": 0.9863505959510803, "metrics/particle/class_2/efficiency": 0.5292096138000488, "metrics/particle/class_2/pid_accuracy": 0.8279221057891846, "metrics/particle/class_3/efficiency": 0.8349313735961914, "metrics/particle/class_3/pid_accuracy": 0.991443395614624, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.5843137502670288, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.8022598624229431, "metrics/particle/count/bias_mean": 0.8240000009536743, "metrics/particle/count/mae": 3.2720000743865967, "metrics/particle/count/prediction_mean": 99.01599884033203, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024525292217731476, "metrics/particle/event/energy_response_mean": 1.0010758638381958, "metrics/particle/event/met_abs_error": 4.053495407104492, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024451270699501038, "metrics/particle/event/scalar_pt_response_mean": 1.0039130449295044, "metrics/particle/event/vector_pt_closure": 0.0267349760979414, "metrics/particle/matched/delta_eta_abs_mean": 0.009046854451298714, "metrics/particle/matched/delta_phi_abs_mean": 0.01030537486076355, "metrics/particle/matched/delta_r_mean": 0.015334317460656166, "metrics/particle/matched/energy_relative_abs_mean": 0.08585821837186813, "metrics/particle/matched/pid_accuracy": 0.9752970933914185, "metrics/particle/matched/pt_relative_abs_mean": 0.08065269887447357, "metrics/particle/matching/duplicate_fraction": 0.02468287944793701, "metrics/particle/matching/efficiency": 0.87070232629776, "metrics/particle/matching/f1": 0.8670642375946045, "metrics/particle/matching/purity": 0.8634564280509949, "metrics/particle/matching/target_coverage_dr0p05": 0.8444679975509644, "metrics/particle/matching/target_coverage_dr0p10": 0.8987290263175964, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7677204012870789, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9001955389976501}, "step_train_time": 0.8485593795776367, "step_valid_time": 2.0311524940188974, "step_total_time": 2185.82639670372, "elapsed_training_loop_seconds": 2185.82639670372, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027215710287598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06722924323948565, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553928095872171, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9550748752079867, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04460719041278296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527296937416777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524126455906822, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547571523619428, "step_test_time": 3.2421257340465672, "step_plot_time": 2.7866153260110877, "step_validation_cycle_time": 8.43135709798662} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..32e70fc87c390620ad9e212a4cea4c055888030e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 1.7658871412277222, "Classification": 0.01062789186835289, "Regression_pt": 0.07174939662218094, "Regression_eta": 0.004564459901303053, "Regression_sin_phi": 0.0046370369382202625, "Regression_cos_phi": 0.002619198290631175, "Regression_energy": 0.06969835609197617, "Total": 10.384347915649414, "Total_unweighted": 1.9297834809403867}, "train_group": {}, "valid": {"Classification_binary": 2.0737195014953613, "Classification": 0.015993179753422737, "Regression_pt": 0.09601564705371857, "Regression_eta": 0.007969286292791367, "Regression_sin_phi": 0.006506649311631918, "Regression_cos_phi": 0.00590142235159874, "Regression_energy": 0.09728020429611206, "Total": 15.601348876953125, "Total_unweighted": 2.3033858905546367, "metrics/particle/class_1/efficiency": 0.9532814025878906, "metrics/particle/class_1/pid_accuracy": 0.9869850277900696, "metrics/particle/class_2/efficiency": 0.5206185579299927, "metrics/particle/class_2/pid_accuracy": 0.8360835909843445, "metrics/particle/class_3/efficiency": 0.8287271857261658, "metrics/particle/class_3/pid_accuracy": 0.9910390377044678, "metrics/particle/class_4/efficiency": 0.8745762705802917, "metrics/particle/class_4/pid_accuracy": 0.604651153087616, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7359550595283508, "metrics/particle/count/bias_mean": 0.6600000262260437, "metrics/particle/count/mae": 3.2279999256134033, "metrics/particle/count/prediction_mean": 98.85199737548828, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02500036731362343, "metrics/particle/event/energy_response_mean": 1.000923752784729, "metrics/particle/event/met_abs_error": 4.213115692138672, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025627082213759422, "metrics/particle/event/scalar_pt_response_mean": 1.000819444656372, "metrics/particle/event/vector_pt_closure": 0.028293199837207794, "metrics/particle/matched/delta_eta_abs_mean": 0.008762393146753311, "metrics/particle/matched/delta_phi_abs_mean": 0.010359136387705803, "metrics/particle/matched/delta_r_mean": 0.015163779258728027, "metrics/particle/matched/energy_relative_abs_mean": 0.08669181913137436, "metrics/particle/matched/pid_accuracy": 0.9754952788352966, "metrics/particle/matched/pt_relative_abs_mean": 0.08084207773208618, "metrics/particle/matching/duplicate_fraction": 0.024885687977075577, "metrics/particle/matching/efficiency": 0.8677692413330078, "metrics/particle/matching/f1": 0.8648626804351807, "metrics/particle/matching/purity": 0.8619754910469055, "metrics/particle/matching/target_coverage_dr0p05": 0.8423904180526733, "metrics/particle/matching/target_coverage_dr0p10": 0.896284818649292, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7640948295593262, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8979957699775696}, "step_train_time": 0.8410983085632324, "step_valid_time": 2.033095497987233, "step_total_time": 2269.3517360687256, "elapsed_training_loop_seconds": 2269.3517360687256, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017393211938588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06555061235697024, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514304723885563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552438209752839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533333333333334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0447561790247161, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9481037924151696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519038076152304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514304723885563, "step_test_time": 3.291495757002849, "step_plot_time": 2.779291936021764, "step_validation_cycle_time": 8.490733302023727} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..8da8a8ccc4fe8799a5d7014de6cce816ea0badc9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 1.9649064540863037, "Classification": 0.011520381085574627, "Regression_pt": 0.06043297424912453, "Regression_eta": 0.0024903679732233286, "Regression_sin_phi": 0.0020640953443944454, "Regression_cos_phi": 0.0037627066485583782, "Regression_energy": 0.05899147316813469, "Total": 9.01651382446289, "Total_unweighted": 2.1041684525553137}, "train_group": {}, "valid": {"Classification_binary": 2.0747756958007812, "Classification": 0.0163124892860651, "Regression_pt": 0.09452812373638153, "Regression_eta": 0.007811376824975014, "Regression_sin_phi": 0.00639520026743412, "Regression_cos_phi": 0.0059462315402925014, "Regression_energy": 0.09600891172885895, "Total": 15.509970664978027, "Total_unweighted": 2.3017780291847885, "metrics/particle/class_1/efficiency": 0.9521690607070923, "metrics/particle/class_1/pid_accuracy": 0.9854421019554138, "metrics/particle/class_2/efficiency": 0.520045816898346, "metrics/particle/class_2/pid_accuracy": 0.8447136282920837, "metrics/particle/class_3/efficiency": 0.8252491354942322, "metrics/particle/class_3/pid_accuracy": 0.990090012550354, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.61328125, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7966101765632629, "metrics/particle/count/bias_mean": 0.5640000104904175, "metrics/particle/count/mae": 3.187999963760376, "metrics/particle/count/prediction_mean": 98.75599670410156, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02618320845067501, "metrics/particle/event/energy_response_mean": 1.0039665699005127, "metrics/particle/event/met_abs_error": 4.189590930938721, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026582248508930206, "metrics/particle/event/scalar_pt_response_mean": 1.003804087638855, "metrics/particle/event/vector_pt_closure": 0.027439115568995476, "metrics/particle/matched/delta_eta_abs_mean": 0.009062457829713821, "metrics/particle/matched/delta_phi_abs_mean": 0.01031103078275919, "metrics/particle/matched/delta_r_mean": 0.01536597404628992, "metrics/particle/matched/energy_relative_abs_mean": 0.08678875863552094, "metrics/particle/matched/pid_accuracy": 0.9752917885780334, "metrics/particle/matched/pt_relative_abs_mean": 0.08081384748220444, "metrics/particle/matching/duplicate_fraction": 0.02685406431555748, "metrics/particle/matching/efficiency": 0.8655694723129272, "metrics/particle/matching/f1": 0.8630907535552979, "metrics/particle/matching/purity": 0.8606261610984802, "metrics/particle/matching/target_coverage_dr0p05": 0.8408424258232117, "metrics/particle/matching/target_coverage_dr0p10": 0.8953478932380676, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7625468373298645, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8966107368469238}, "step_train_time": 0.8227958679199219, "step_valid_time": 2.0455904430127703, "step_total_time": 2354.3572964668274, "elapsed_training_loop_seconds": 2354.3572964668274, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030203712754406, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0689990041774653, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9532710280373832, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.951682772409197, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04672897196261682, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9499332443257676, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9483505498167277, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9500998003992016, "step_test_time": 3.568892292038072, "step_plot_time": 2.8707682290114462, "step_validation_cycle_time": 8.885334142949432} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..cd5bda115c6bcabd231aa119a87fad64952c054c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 1.574692726135254, "Classification": 0.010934722609817982, "Regression_pt": 0.06927294284105301, "Regression_eta": 0.005710447207093239, "Regression_sin_phi": 0.003811850678175688, "Regression_cos_phi": 0.003243307350203395, "Regression_energy": 0.06819944828748703, "Total": 10.457633972167969, "Total_unweighted": 1.7358654451090842}, "train_group": {}, "valid": {"Classification_binary": 2.0833539962768555, "Classification": 0.01656649261713028, "Regression_pt": 0.09581374377012253, "Regression_eta": 0.007738820742815733, "Regression_sin_phi": 0.006338817533105612, "Regression_cos_phi": 0.005965071264654398, "Regression_energy": 0.09753129631280899, "Total": 15.57041072845459, "Total_unweighted": 2.313308238517493, "metrics/particle/class_1/efficiency": 0.9520835280418396, "metrics/particle/class_1/pid_accuracy": 0.9885863065719604, "metrics/particle/class_2/efficiency": 0.5246277451515198, "metrics/particle/class_2/pid_accuracy": 0.8427947759628296, "metrics/particle/class_3/efficiency": 0.8369054198265076, "metrics/particle/class_3/pid_accuracy": 0.9909019470214844, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.5764706134796143, "metrics/particle/class_5/efficiency": 0.9835164546966553, "metrics/particle/class_5/pid_accuracy": 0.7709497213363647, "metrics/particle/count/bias_mean": 0.9760000109672546, "metrics/particle/count/mae": 3.2799999713897705, "metrics/particle/count/prediction_mean": 99.16799926757812, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.025934336706995964, "metrics/particle/event/energy_response_mean": 1.0024412870407104, "metrics/particle/event/met_abs_error": 4.285913944244385, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026220234110951424, "metrics/particle/event/scalar_pt_response_mean": 1.0041368007659912, "metrics/particle/event/vector_pt_closure": 0.02871384471654892, "metrics/particle/matched/delta_eta_abs_mean": 0.008816364221274853, "metrics/particle/matched/delta_phi_abs_mean": 0.01053065899759531, "metrics/particle/matched/delta_r_mean": 0.015362191945314407, "metrics/particle/matched/energy_relative_abs_mean": 0.08651810139417648, "metrics/particle/matched/pid_accuracy": 0.9765669107437134, "metrics/particle/matched/pt_relative_abs_mean": 0.0810474082827568, "metrics/particle/matching/duplicate_fraction": 0.0255727656185627, "metrics/particle/matching/efficiency": 0.8709467053413391, "metrics/particle/matching/f1": 0.8666396141052246, "metrics/particle/matching/purity": 0.8623749613761902, "metrics/particle/matching/target_coverage_dr0p05": 0.8423497080802917, "metrics/particle/matching/target_coverage_dr0p10": 0.8977513313293457, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7688609957695007, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.900439977645874}, "step_train_time": 0.8353333473205566, "step_valid_time": 2.006520392955281, "step_total_time": 2438.0303020477295, "elapsed_training_loop_seconds": 2438.0303020477295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022836730272033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06499785634745181, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.957418496340652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9612558450233801, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9593333333333334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038744154976619906, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585838343353373, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9566666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.957418496340652, "step_test_time": 3.250512363971211, "step_plot_time": 2.825427217001561, "step_validation_cycle_time": 8.461646210984327} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..9041afa448ad74ccfe66452c8e2db44c36ccb835 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 1.961973786354065, "Classification": 0.014409385621547699, "Regression_pt": 0.08429989963769913, "Regression_eta": 0.0025516210589557886, "Regression_sin_phi": 0.0029098596423864365, "Regression_cos_phi": 0.0040135509334504604, "Regression_energy": 0.08274317532777786, "Total": 10.809001922607422, "Total_unweighted": 2.1529012785758823}, "train_group": {}, "valid": {"Classification_binary": 1.985013484954834, "Classification": 0.014246814884245396, "Regression_pt": 0.08699721097946167, "Regression_eta": 0.00690119806677103, "Regression_sin_phi": 0.005525798536837101, "Regression_cos_phi": 0.00527555774897337, "Regression_energy": 0.08743847161531448, "Total": 13.917945861816406, "Total_unweighted": 2.191398536786437, "metrics/particle/class_1/efficiency": 0.9502010941505432, "metrics/particle/class_1/pid_accuracy": 0.9883835911750793, "metrics/particle/class_2/efficiency": 0.5406643748283386, "metrics/particle/class_2/pid_accuracy": 0.8463982939720154, "metrics/particle/class_3/efficiency": 0.835119366645813, "metrics/particle/class_3/pid_accuracy": 0.9903196692466736, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.5868725776672363, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7897727489471436, "metrics/particle/count/bias_mean": 0.41200000047683716, "metrics/particle/count/mae": 3.2279999256134033, "metrics/particle/count/prediction_mean": 98.60399627685547, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02612384594976902, "metrics/particle/event/energy_response_mean": 1.0046741962432861, "metrics/particle/event/met_abs_error": 3.9884815216064453, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02513309009373188, "metrics/particle/event/scalar_pt_response_mean": 1.0055303573608398, "metrics/particle/event/vector_pt_closure": 0.02626859024167061, "metrics/particle/matched/delta_eta_abs_mean": 0.00845397263765335, "metrics/particle/matched/delta_phi_abs_mean": 0.009790746495127678, "metrics/particle/matched/delta_r_mean": 0.014455344527959824, "metrics/particle/matched/energy_relative_abs_mean": 0.08708951622247696, "metrics/particle/matched/pid_accuracy": 0.9764133095741272, "metrics/particle/matched/pt_relative_abs_mean": 0.08148171752691269, "metrics/particle/matching/duplicate_fraction": 0.02515110932290554, "metrics/particle/matching/efficiency": 0.8704578876495361, "metrics/particle/matching/f1": 0.8686355352401733, "metrics/particle/matching/purity": 0.8668208122253418, "metrics/particle/matching/target_coverage_dr0p05": 0.8479713201522827, "metrics/particle/matching/target_coverage_dr0p10": 0.8980364799499512, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7653576731681824, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9008066058158875}, "step_train_time": 0.8528106212615967, "step_valid_time": 2.078953152988106, "step_total_time": 246.52419304847717, "elapsed_training_loop_seconds": 246.52419304847717, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0044030093255045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058546299712934236, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1506, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.950199203187251, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9511465603190429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049800796812749, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9482071713147411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9491525423728814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.190588152036071, "step_plot_time": 2.7460226300172508, "step_validation_cycle_time": 8.42522854503477} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..67acd44129996113af429a4050f4b851f63cb69d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.8594903945922852, "Classification": 0.011833678930997849, "Regression_pt": 0.06224248930811882, "Regression_eta": 0.0052495598793029785, "Regression_sin_phi": 0.002016089390963316, "Regression_cos_phi": 0.00285928207449615, "Regression_energy": 0.0615120604634285, "Total": 9.577349662780762, "Total_unweighted": 2.0052035546395928}, "train_group": {}, "valid": {"Classification_binary": 2.093236207962036, "Classification": 0.01675676926970482, "Regression_pt": 0.09741497039794922, "Regression_eta": 0.0080656036734581, "Regression_sin_phi": 0.006524355616420507, "Regression_cos_phi": 0.005963312927633524, "Regression_energy": 0.09904728829860687, "Total": 15.848237991333008, "Total_unweighted": 2.327008508145809, "metrics/particle/class_1/efficiency": 0.9516556859016418, "metrics/particle/class_1/pid_accuracy": 0.987232506275177, "metrics/particle/class_2/efficiency": 0.5189003348350525, "metrics/particle/class_2/pid_accuracy": 0.8322295546531677, "metrics/particle/class_3/efficiency": 0.8325813412666321, "metrics/particle/class_3/pid_accuracy": 0.9904030561447144, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.5797665119171143, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7808988690376282, "metrics/particle/count/bias_mean": 0.29600000381469727, "metrics/particle/count/mae": 3.2320001125335693, "metrics/particle/count/prediction_mean": 98.48799896240234, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.027064034715294838, "metrics/particle/event/energy_response_mean": 0.9967221617698669, "metrics/particle/event/met_abs_error": 4.276175498962402, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027497094124555588, "metrics/particle/event/scalar_pt_response_mean": 0.997086226940155, "metrics/particle/event/vector_pt_closure": 0.027942290529608727, "metrics/particle/matched/delta_eta_abs_mean": 0.008793472312390804, "metrics/particle/matched/delta_phi_abs_mean": 0.011037633754312992, "metrics/particle/matched/delta_r_mean": 0.015751469880342484, "metrics/particle/matched/energy_relative_abs_mean": 0.08690031617879868, "metrics/particle/matched/pid_accuracy": 0.9753283262252808, "metrics/particle/matched/pt_relative_abs_mean": 0.0802759975194931, "metrics/particle/matching/duplicate_fraction": 0.024774592369794846, "metrics/particle/matching/efficiency": 0.8685024976730347, "metrics/particle/matching/f1": 0.8671954274177551, "metrics/particle/matching/purity": 0.8658922910690308, "metrics/particle/matching/target_coverage_dr0p05": 0.8389685750007629, "metrics/particle/matching/target_coverage_dr0p10": 0.8953071236610413, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7665390372276306, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8979142904281616}, "step_train_time": 0.827003002166748, "step_valid_time": 2.0450626449892297, "step_total_time": 2523.240898132324, "elapsed_training_loop_seconds": 2523.240898132324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9989857995838877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0675925475945649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9540918163672655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9585561497326203, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9563187729243081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04144385026737968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9514304723885563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9558823529411765, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536512170723574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9540918163672655, "step_test_time": 3.2654033720027655, "step_plot_time": 2.794165024999529, "step_validation_cycle_time": 8.49995276296977} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..7d2dd3967be3d3858815e145ba3fecdb0256f03a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 1.9460299015045166, "Classification": 0.015077909454703331, "Regression_pt": 0.08396994322538376, "Regression_eta": 0.008116123266518116, "Regression_sin_phi": 0.0036993639077991247, "Regression_cos_phi": 0.0031159722711890936, "Regression_energy": 0.08361482620239258, "Total": 12.685035705566406, "Total_unweighted": 2.1436240398325026}, "train_group": {}, "valid": {"Classification_binary": 2.08872652053833, "Classification": 0.01683006063103676, "Regression_pt": 0.09824001789093018, "Regression_eta": 0.007995810359716415, "Regression_sin_phi": 0.006483104079961777, "Regression_cos_phi": 0.0058936020359396935, "Regression_energy": 0.09991365671157837, "Total": 15.82102108001709, "Total_unweighted": 2.3240827722474933, "metrics/particle/class_1/efficiency": 0.9540514945983887, "metrics/particle/class_1/pid_accuracy": 0.9873542785644531, "metrics/particle/class_2/efficiency": 0.524055004119873, "metrics/particle/class_2/pid_accuracy": 0.8360655903816223, "metrics/particle/class_3/efficiency": 0.831453263759613, "metrics/particle/class_3/pid_accuracy": 0.9908422827720642, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.5680933594703674, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.7752808928489685, "metrics/particle/count/bias_mean": 0.6200000047683716, "metrics/particle/count/mae": 3.2360000610351562, "metrics/particle/count/prediction_mean": 98.81199645996094, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.028149480000138283, "metrics/particle/event/energy_response_mean": 0.9973715543746948, "metrics/particle/event/met_abs_error": 4.35115909576416, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02767491526901722, "metrics/particle/event/scalar_pt_response_mean": 0.9981980919837952, "metrics/particle/event/vector_pt_closure": 0.028882762417197227, "metrics/particle/matched/delta_eta_abs_mean": 0.00905847828835249, "metrics/particle/matched/delta_phi_abs_mean": 0.010419533587992191, "metrics/particle/matched/delta_r_mean": 0.01544186845421791, "metrics/particle/matched/energy_relative_abs_mean": 0.08600252121686935, "metrics/particle/matched/pid_accuracy": 0.9754977822303772, "metrics/particle/matched/pt_relative_abs_mean": 0.08040116727352142, "metrics/particle/matching/duplicate_fraction": 0.02432902902364731, "metrics/particle/matching/efficiency": 0.8695209622383118, "metrics/particle/matching/f1": 0.8667844533920288, "metrics/particle/matching/purity": 0.8640651106834412, "metrics/particle/matching/target_coverage_dr0p05": 0.8412090539932251, "metrics/particle/matching/target_coverage_dr0p10": 0.8973032236099243, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7675574421882629, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8986882567405701}, "step_train_time": 0.8826966285705566, "step_valid_time": 2.0738679739879444, "step_total_time": 2608.1184451580048, "elapsed_training_loop_seconds": 2608.1184451580048, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9994904946136598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06779809799953629, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514304723885563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9584450402144772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549248747913188, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04155495978552279, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.948769128409847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9557640750670241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522537562604341, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514304723885563, "step_test_time": 3.493738873046823, "step_plot_time": 2.9040375059703365, "step_validation_cycle_time": 8.884297055948991} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_32000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..754057da01073fab3e931ab601b5c4e27e5aee1b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 56.93950177935943, "train": {"Classification_binary": 2.2403106689453125, "Classification": 0.011766462586820126, "Regression_pt": 0.08641678094863892, "Regression_eta": 0.003383844858035445, "Regression_sin_phi": 0.006188818719238043, "Regression_cos_phi": 0.0036823730915784836, "Regression_energy": 0.08624245226383209, "Total": 12.276643753051758, "Total_unweighted": 2.4379914014134556}, "train_group": {}, "valid": {"Classification_binary": 2.1004974842071533, "Classification": 0.01700248010456562, "Regression_pt": 0.09913396090269089, "Regression_eta": 0.008336294442415237, "Regression_sin_phi": 0.006438948214054108, "Regression_cos_phi": 0.005765343550592661, "Regression_energy": 0.1003417894244194, "Total": 15.926972389221191, "Total_unweighted": 2.3375163008458912, "metrics/particle/class_1/efficiency": 0.9513134360313416, "metrics/particle/class_1/pid_accuracy": 0.9849793314933777, "metrics/particle/class_2/efficiency": 0.517754852771759, "metrics/particle/class_2/pid_accuracy": 0.8351770043373108, "metrics/particle/class_3/efficiency": 0.8323932886123657, "metrics/particle/class_3/pid_accuracy": 0.9905138611793518, "metrics/particle/class_4/efficiency": 0.8847457766532898, "metrics/particle/class_4/pid_accuracy": 0.6321839094161987, "metrics/particle/class_5/efficiency": 0.9780219793319702, "metrics/particle/class_5/pid_accuracy": 0.8089887499809265, "metrics/particle/count/bias_mean": 0.46799999475479126, "metrics/particle/count/mae": 3.24399995803833, "metrics/particle/count/prediction_mean": 98.66000366210938, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.027339421212673187, "metrics/particle/event/energy_response_mean": 1.0033458471298218, "metrics/particle/event/met_abs_error": 4.1880059242248535, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027735726907849312, "metrics/particle/event/scalar_pt_response_mean": 1.0045009851455688, "metrics/particle/event/vector_pt_closure": 0.028501305729150772, "metrics/particle/matched/delta_eta_abs_mean": 0.009367816150188446, "metrics/particle/matched/delta_phi_abs_mean": 0.010533123277127743, "metrics/particle/matched/delta_r_mean": 0.01575932465493679, "metrics/particle/matched/energy_relative_abs_mean": 0.08773841708898544, "metrics/particle/matched/pid_accuracy": 0.9751360416412354, "metrics/particle/matched/pt_relative_abs_mean": 0.08141086250543594, "metrics/particle/matching/duplicate_fraction": 0.024690857157111168, "metrics/particle/matching/efficiency": 0.8683395981788635, "metrics/particle/matching/f1": 0.8662751913070679, "metrics/particle/matching/purity": 0.8642205595970154, "metrics/particle/matching/target_coverage_dr0p05": 0.840883195400238, "metrics/particle/matching/target_coverage_dr0p10": 0.8970995545387268, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7632800936698914, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8966107368469238}, "step_train_time": 0.9483230113983154, "step_valid_time": 1.9957448840141296, "step_total_time": 2693.3518331050873, "elapsed_training_loop_seconds": 2693.3518331050873, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005367523250843, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06823477884468132, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9546364242828552, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9533644237175216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04536357571714476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.95263509006004, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9513657561625583, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.459059001004789, "step_plot_time": 2.8403235750156455, "step_validation_cycle_time": 8.670479656022508} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_33000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..f9bd03845c1240f7a84318701e15eb53f425931e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 58.718861209964416, "train": {"Classification_binary": 1.7040568590164185, "Classification": 0.008695886470377445, "Regression_pt": 0.05878927558660507, "Regression_eta": 0.002125767758116126, "Regression_sin_phi": 0.0016289465129375458, "Regression_cos_phi": 0.001758634694851935, "Regression_energy": 0.05871783196926117, "Total": 7.265348434448242, "Total_unweighted": 1.8357732020085678}, "train_group": {}, "valid": {"Classification_binary": 2.102978229522705, "Classification": 0.017674004659056664, "Regression_pt": 0.09861542284488678, "Regression_eta": 0.00823262333869934, "Regression_sin_phi": 0.00659298337996006, "Regression_cos_phi": 0.006036835256963968, "Regression_energy": 0.1000676229596138, "Total": 16.133907318115234, "Total_unweighted": 2.3401977219618857, "metrics/particle/class_1/efficiency": 0.9502866268157959, "metrics/particle/class_1/pid_accuracy": 0.9862236380577087, "metrics/particle/class_2/efficiency": 0.517754852771759, "metrics/particle/class_2/pid_accuracy": 0.8407079577445984, "metrics/particle/class_3/efficiency": 0.8279751539230347, "metrics/particle/class_3/pid_accuracy": 0.990009069442749, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.6015625, "metrics/particle/class_5/efficiency": 0.9835164546966553, "metrics/particle/class_5/pid_accuracy": 0.7821229100227356, "metrics/particle/count/bias_mean": 0.5199999809265137, "metrics/particle/count/mae": 3.319999933242798, "metrics/particle/count/prediction_mean": 98.71199798583984, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.026522209867835045, "metrics/particle/event/energy_response_mean": 1.0034816265106201, "metrics/particle/event/met_abs_error": 4.193838596343994, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026639629155397415, "metrics/particle/event/scalar_pt_response_mean": 1.0047435760498047, "metrics/particle/event/vector_pt_closure": 0.02794155664741993, "metrics/particle/matched/delta_eta_abs_mean": 0.009093617089092731, "metrics/particle/matched/delta_phi_abs_mean": 0.01050885021686554, "metrics/particle/matched/delta_r_mean": 0.015537770465016365, "metrics/particle/matched/energy_relative_abs_mean": 0.0872088372707367, "metrics/particle/matched/pid_accuracy": 0.9752505421638489, "metrics/particle/matched/pt_relative_abs_mean": 0.08084254711866379, "metrics/particle/matching/duplicate_fraction": 0.02601507492363453, "metrics/particle/matching/efficiency": 0.8657732009887695, "metrics/particle/matching/f1": 0.8634867668151855, "metrics/particle/matching/purity": 0.861212432384491, "metrics/particle/matching/target_coverage_dr0p05": 0.8390499949455261, "metrics/particle/matching/target_coverage_dr0p10": 0.8953886032104492, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7647466063499451, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8957552313804626}, "step_train_time": 0.87257981300354, "step_valid_time": 2.003009613021277, "step_total_time": 2778.007875442505, "elapsed_training_loop_seconds": 2778.007875442505, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031493306613566, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06544527587983806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1499, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9534264803725881, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559706470980653, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546968687541639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044029352901934625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507651363938789, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533022014676451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9520319786808794, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9534264803725881, "step_test_time": 3.2831461740424857, "step_plot_time": 2.8028934000176378, "step_validation_cycle_time": 8.482813633047044} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..8a288dff44ddeae1aef482e8b79314562aeec106 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 1.6090840101242065, "Classification": 0.014868127182126045, "Regression_pt": 0.0876067653298378, "Regression_eta": 0.003432126250118017, "Regression_sin_phi": 0.0051479460671544075, "Regression_cos_phi": 0.005056220106780529, "Regression_energy": 0.0868547335267067, "Total": 12.241499900817871, "Total_unweighted": 1.81204992858693}, "train_group": {}, "valid": {"Classification_binary": 1.982576608657837, "Classification": 0.014233105815947056, "Regression_pt": 0.08766669780015945, "Regression_eta": 0.00699611147865653, "Regression_sin_phi": 0.005478573497384787, "Regression_cos_phi": 0.0052802120335400105, "Regression_energy": 0.08813627064228058, "Total": 13.956476211547852, "Total_unweighted": 2.1903675799258053, "metrics/particle/class_1/efficiency": 0.9505433440208435, "metrics/particle/class_1/pid_accuracy": 0.9885678291320801, "metrics/particle/class_2/efficiency": 0.5418098568916321, "metrics/particle/class_2/pid_accuracy": 0.8446089029312134, "metrics/particle/class_3/efficiency": 0.8354954123497009, "metrics/particle/class_3/pid_accuracy": 0.9911116361618042, "metrics/particle/class_4/efficiency": 0.8745762705802917, "metrics/particle/class_4/pid_accuracy": 0.6162790656089783, "metrics/particle/class_5/efficiency": 0.9725274443626404, "metrics/particle/class_5/pid_accuracy": 0.7853107452392578, "metrics/particle/count/bias_mean": 0.46000000834465027, "metrics/particle/count/mae": 3.140000104904175, "metrics/particle/count/prediction_mean": 98.6520004272461, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02484762854874134, "metrics/particle/event/energy_response_mean": 0.99773108959198, "metrics/particle/event/met_abs_error": 4.180314064025879, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02378825657069683, "metrics/particle/event/scalar_pt_response_mean": 0.9994152784347534, "metrics/particle/event/vector_pt_closure": 0.026509050279855728, "metrics/particle/matched/delta_eta_abs_mean": 0.008501092903316021, "metrics/particle/matched/delta_phi_abs_mean": 0.009702886454761028, "metrics/particle/matched/delta_r_mean": 0.014404923655092716, "metrics/particle/matched/energy_relative_abs_mean": 0.08554323762655258, "metrics/particle/matched/pid_accuracy": 0.9770792126655579, "metrics/particle/matched/pt_relative_abs_mean": 0.08081334829330444, "metrics/particle/matching/duplicate_fraction": 0.02521996572613716, "metrics/particle/matching/efficiency": 0.8708652257919312, "metrics/particle/matching/f1": 0.868830144405365, "metrics/particle/matching/purity": 0.8668045401573181, "metrics/particle/matching/target_coverage_dr0p05": 0.8487452864646912, "metrics/particle/matching/target_coverage_dr0p10": 0.8990548849105835, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7671908140182495, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9012547135353088}, "step_train_time": 0.7968640327453613, "step_valid_time": 2.046770765038673, "step_total_time": 330.8012385368347, "elapsed_training_loop_seconds": 330.8012385368347, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998919307745304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05807591805257317, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9514304723885563, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507978723404256, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.951114067176588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04920212765957447, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9474384564204924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9468085106382979, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.94712337878284, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9514304723885563, "step_test_time": 3.203294039994944, "step_plot_time": 2.762587663019076, "step_validation_cycle_time": 8.364239498972893} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a993e0190096d559d1a6b6bbcc58c77e8392b1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 2.0676352977752686, "Classification": 0.012058345600962639, "Regression_pt": 0.08955283463001251, "Regression_eta": 0.008417105302214622, "Regression_sin_phi": 0.005537217017263174, "Regression_cos_phi": 0.005509424023330212, "Regression_energy": 0.0880335122346878, "Total": 14.378031730651855, "Total_unweighted": 2.2767437365837395}, "train_group": {}, "valid": {"Classification_binary": 1.9865154027938843, "Classification": 0.014348689466714859, "Regression_pt": 0.08767158538103104, "Regression_eta": 0.006927323527634144, "Regression_sin_phi": 0.005546989850699902, "Regression_cos_phi": 0.00530218193307519, "Regression_energy": 0.08821555227041245, "Total": 13.989736557006836, "Total_unweighted": 2.194527725223452, "metrics/particle/class_1/efficiency": 0.9511423110961914, "metrics/particle/class_1/pid_accuracy": 0.989384651184082, "metrics/particle/class_2/efficiency": 0.5435280799865723, "metrics/particle/class_2/pid_accuracy": 0.8472076058387756, "metrics/particle/class_3/efficiency": 0.8308892846107483, "metrics/particle/class_3/pid_accuracy": 0.9916279911994934, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.5945945978164673, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7897727489471436, "metrics/particle/count/bias_mean": 0.4959999918937683, "metrics/particle/count/mae": 3.1679999828338623, "metrics/particle/count/prediction_mean": 98.68800354003906, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02397967502474785, "metrics/particle/event/energy_response_mean": 1.0033119916915894, "metrics/particle/event/met_abs_error": 4.2477641105651855, "metrics/particle/event/scalar_pt_relative_abs_error": 0.023723581805825233, "metrics/particle/event/scalar_pt_response_mean": 1.0066523551940918, "metrics/particle/event/vector_pt_closure": 0.026212017983198166, "metrics/particle/matched/delta_eta_abs_mean": 0.008703840896487236, "metrics/particle/matched/delta_phi_abs_mean": 0.009618810378015041, "metrics/particle/matched/delta_r_mean": 0.014506928622722626, "metrics/particle/matched/energy_relative_abs_mean": 0.08605016022920609, "metrics/particle/matched/pid_accuracy": 0.9775528311729431, "metrics/particle/matched/pt_relative_abs_mean": 0.08090236037969589, "metrics/particle/matching/duplicate_fraction": 0.025494487956166267, "metrics/particle/matching/efficiency": 0.8692765235900879, "metrics/particle/matching/f1": 0.8670865297317505, "metrics/particle/matching/purity": 0.8649075627326965, "metrics/particle/matching/target_coverage_dr0p05": 0.846586287021637, "metrics/particle/matching/target_coverage_dr0p10": 0.8975883722305298, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7659687399864197, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9006028771400452}, "step_train_time": 0.8299248218536377, "step_valid_time": 1.9802043169620447, "step_total_time": 415.1025960445404, "elapsed_training_loop_seconds": 415.1025960445404, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00383113079335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059246708240738855, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9547571523619428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04524284763805722, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547571523619428, "step_test_time": 3.6529323720023967, "step_plot_time": 2.7475274939788505, "step_validation_cycle_time": 8.731228939024732} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..7276259af7eb234685997d25d8c754c1277c0941 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 2.1205220222473145, "Classification": 0.01608157344162464, "Regression_pt": 0.09925857931375504, "Regression_eta": 0.005643058568239212, "Regression_sin_phi": 0.003096547443419695, "Regression_cos_phi": 0.004589289892464876, "Regression_energy": 0.10156480967998505, "Total": 13.197793960571289, "Total_unweighted": 2.350755880586803}, "train_group": {}, "valid": {"Classification_binary": 1.9896483421325684, "Classification": 0.014350270852446556, "Regression_pt": 0.08833920210599899, "Regression_eta": 0.006972677540034056, "Regression_sin_phi": 0.005561992526054382, "Regression_cos_phi": 0.005283225327730179, "Regression_energy": 0.08892156183719635, "Total": 14.034412384033203, "Total_unweighted": 2.199077272322029, "metrics/particle/class_1/efficiency": 0.9521690607070923, "metrics/particle/class_1/pid_accuracy": 0.9887670874595642, "metrics/particle/class_2/efficiency": 0.5400916337966919, "metrics/particle/class_2/pid_accuracy": 0.847295880317688, "metrics/particle/class_3/efficiency": 0.8311712741851807, "metrics/particle/class_3/pid_accuracy": 0.99140465259552, "metrics/particle/class_4/efficiency": 0.8813559412956238, "metrics/particle/class_4/pid_accuracy": 0.6115384697914124, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7784090638160706, "metrics/particle/count/bias_mean": 0.3799999952316284, "metrics/particle/count/mae": 3.0840001106262207, "metrics/particle/count/prediction_mean": 98.5719985961914, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02423788607120514, "metrics/particle/event/energy_response_mean": 1.0014514923095703, "metrics/particle/event/met_abs_error": 4.05398416519165, "metrics/particle/event/scalar_pt_relative_abs_error": 0.023758284747600555, "metrics/particle/event/scalar_pt_response_mean": 1.0027623176574707, "metrics/particle/event/vector_pt_closure": 0.025623034685850143, "metrics/particle/matched/delta_eta_abs_mean": 0.00827393215149641, "metrics/particle/matched/delta_phi_abs_mean": 0.009685489349067211, "metrics/particle/matched/delta_r_mean": 0.014230334199965, "metrics/particle/matched/energy_relative_abs_mean": 0.08601542562246323, "metrics/particle/matched/pid_accuracy": 0.9772822856903076, "metrics/particle/matched/pt_relative_abs_mean": 0.08072910457849503, "metrics/particle/matching/duplicate_fraction": 0.024428844451904297, "metrics/particle/matching/efficiency": 0.8696838617324829, "metrics/particle/matching/f1": 0.868004322052002, "metrics/particle/matching/purity": 0.8663312196731567, "metrics/particle/matching/target_coverage_dr0p05": 0.8492748737335205, "metrics/particle/matching/target_coverage_dr0p10": 0.898769736289978, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7656835317611694, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8999918699264526}, "step_train_time": 0.8144066333770752, "step_valid_time": 2.021239354973659, "step_total_time": 498.966792345047, "elapsed_training_loop_seconds": 498.966792345047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022499773718383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060058722621385896, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9527296937416777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524126455906822, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04727030625832224, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.948769128409847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9494007989347537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490848585690516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.2107258299947716, "step_plot_time": 2.7489525609998964, "step_validation_cycle_time": 8.338342790026218} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..bc7b019973d2f6f4bc3f33a805a32c47da1083cd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 1.8820924758911133, "Classification": 0.015242766588926315, "Regression_pt": 0.08420994877815247, "Regression_eta": 0.00329817202873528, "Regression_sin_phi": 0.005104220937937498, "Regression_cos_phi": 0.004002549219876528, "Regression_energy": 0.08159230649471283, "Total": 11.908541679382324, "Total_unweighted": 2.075542439939454}, "train_group": {}, "valid": {"Classification_binary": 1.9909826517105103, "Classification": 0.01431259885430336, "Regression_pt": 0.08789318054914474, "Regression_eta": 0.006987923756241798, "Regression_sin_phi": 0.005599739961326122, "Regression_cos_phi": 0.0053536309860646725, "Regression_energy": 0.08858181536197662, "Total": 14.062414169311523, "Total_unweighted": 2.1997115411795676, "metrics/particle/class_1/efficiency": 0.9525113105773926, "metrics/particle/class_1/pid_accuracy": 0.9882321357727051, "metrics/particle/class_2/efficiency": 0.5458190441131592, "metrics/particle/class_2/pid_accuracy": 0.8488982319831848, "metrics/particle/class_3/efficiency": 0.838315486907959, "metrics/particle/class_3/pid_accuracy": 0.9915900230407715, "metrics/particle/class_4/efficiency": 0.8847457766532898, "metrics/particle/class_4/pid_accuracy": 0.6091954112052917, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.7885714173316956, "metrics/particle/count/bias_mean": 0.6159999966621399, "metrics/particle/count/mae": 3.1679999828338623, "metrics/particle/count/prediction_mean": 98.80799865722656, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.024417290464043617, "metrics/particle/event/energy_response_mean": 0.9991438388824463, "metrics/particle/event/met_abs_error": 3.8887317180633545, "metrics/particle/event/scalar_pt_relative_abs_error": 0.023654639720916748, "metrics/particle/event/scalar_pt_response_mean": 1.0001916885375977, "metrics/particle/event/vector_pt_closure": 0.025945009663701057, "metrics/particle/matched/delta_eta_abs_mean": 0.008327183313667774, "metrics/particle/matched/delta_phi_abs_mean": 0.009856569580733776, "metrics/particle/matched/delta_r_mean": 0.01443182211369276, "metrics/particle/matched/energy_relative_abs_mean": 0.08560574054718018, "metrics/particle/matched/pid_accuracy": 0.9771910905838013, "metrics/particle/matched/pt_relative_abs_mean": 0.08081391453742981, "metrics/particle/matching/duplicate_fraction": 0.024653874337673187, "metrics/particle/matching/efficiency": 0.8733501434326172, "metrics/particle/matching/f1": 0.8706192970275879, "metrics/particle/matching/purity": 0.8679054379463196, "metrics/particle/matching/target_coverage_dr0p05": 0.8510265350341797, "metrics/particle/matching/target_coverage_dr0p10": 0.9003177285194397, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7697979211807251, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9041469693183899}, "step_train_time": 0.8277685642242432, "step_valid_time": 2.04958540701773, "step_total_time": 582.8451085090637, "elapsed_training_loop_seconds": 582.8451085090637, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001627988337605, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05835405264273774, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1497, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520958083832335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559118236472945, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04408817635270541, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.948769128409847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9525718102872411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9506666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520958083832335, "step_test_time": 3.706307480984833, "step_plot_time": 2.790110188012477, "step_validation_cycle_time": 8.92285642400384} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..d1c396b7e1005800add64bfcadd571501782230b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 2.3900163173675537, "Classification": 0.013119248673319817, "Regression_pt": 0.08031638711690903, "Regression_eta": 0.002792067127302289, "Regression_sin_phi": 0.0038515287451446056, "Regression_cos_phi": 0.003862409619614482, "Regression_energy": 0.08145713061094284, "Total": 11.347508430480957, "Total_unweighted": 2.5754150892607868}, "train_group": {}, "valid": {"Classification_binary": 1.9980337619781494, "Classification": 0.014415327459573746, "Regression_pt": 0.08887968212366104, "Regression_eta": 0.007078834343701601, "Regression_sin_phi": 0.0055944849736988544, "Regression_cos_phi": 0.005313494708389044, "Regression_energy": 0.08952220529317856, "Total": 14.134285926818848, "Total_unweighted": 2.2088377908803523, "metrics/particle/class_1/efficiency": 0.9512278437614441, "metrics/particle/class_1/pid_accuracy": 0.9899253249168396, "metrics/particle/class_2/efficiency": 0.5360824465751648, "metrics/particle/class_2/pid_accuracy": 0.8482906222343445, "metrics/particle/class_3/efficiency": 0.8363414406776428, "metrics/particle/class_3/pid_accuracy": 0.9911205768585205, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.575875461101532, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.7942857146263123, "metrics/particle/count/bias_mean": 0.5120000243186951, "metrics/particle/count/mae": 3.119999885559082, "metrics/particle/count/prediction_mean": 98.7040023803711, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.0229856688529253, "metrics/particle/event/energy_response_mean": 1.000221848487854, "metrics/particle/event/met_abs_error": 4.212257385253906, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02273741364479065, "metrics/particle/event/scalar_pt_response_mean": 1.0018112659454346, "metrics/particle/event/vector_pt_closure": 0.02673783153295517, "metrics/particle/matched/delta_eta_abs_mean": 0.008365846239030361, "metrics/particle/matched/delta_phi_abs_mean": 0.009730183519423008, "metrics/particle/matched/delta_r_mean": 0.014335273765027523, "metrics/particle/matched/energy_relative_abs_mean": 0.08574492484331131, "metrics/particle/matched/pid_accuracy": 0.9776447415351868, "metrics/particle/matched/pt_relative_abs_mean": 0.0808510035276413, "metrics/particle/matching/duplicate_fraction": 0.02496352791786194, "metrics/particle/matching/efficiency": 0.8710281848907471, "metrics/particle/matching/f1": 0.8687632083892822, "metrics/particle/matching/purity": 0.8665099740028381, "metrics/particle/matching/target_coverage_dr0p05": 0.8511894941329956, "metrics/particle/matching/target_coverage_dr0p10": 0.8995844721794128, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7665797472000122, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9003177285194397}, "step_train_time": 0.8476831912994385, "step_valid_time": 2.0199596079764888, "step_total_time": 667.7400345802307, "elapsed_training_loop_seconds": 667.7400345802307, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012502603494386, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05944033467678955, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9500998003992016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9513657561625583, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9507323568575233, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04863424383744171, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467731204258151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480346435709527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9474034620505992, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9500998003992016, "step_test_time": 3.7344702149857767, "step_plot_time": 2.7522025909856893, "step_validation_cycle_time": 8.855656636005733} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..5ac872966ffb087c5d2ae06dc914dcb51b852f8a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 1.9737101793289185, "Classification": 0.01563381589949131, "Regression_pt": 0.08854743838310242, "Regression_eta": 0.004710561130195856, "Regression_sin_phi": 0.004656494595110416, "Regression_cos_phi": 0.005583415273576975, "Regression_energy": 0.08834227174520493, "Total": 13.196167945861816, "Total_unweighted": 2.1811841763556004}, "train_group": {}, "valid": {"Classification_binary": 2.0023858547210693, "Classification": 0.014430297538638115, "Regression_pt": 0.08908054232597351, "Regression_eta": 0.00714421272277832, "Regression_sin_phi": 0.005629522725939751, "Regression_cos_phi": 0.005361449904739857, "Regression_energy": 0.08985628187656403, "Total": 14.205737113952637, "Total_unweighted": 2.213888161815703, "metrics/particle/class_1/efficiency": 0.951826810836792, "metrics/particle/class_1/pid_accuracy": 0.9899317026138306, "metrics/particle/class_2/efficiency": 0.5389461517333984, "metrics/particle/class_2/pid_accuracy": 0.846971333026886, "metrics/particle/class_3/efficiency": 0.8338033556938171, "metrics/particle/class_3/pid_accuracy": 0.9900789260864258, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.55859375, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.7954545617103577, "metrics/particle/count/bias_mean": 0.6759999990463257, "metrics/particle/count/mae": 3.1640000343322754, "metrics/particle/count/prediction_mean": 98.86799621582031, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.02435119077563286, "metrics/particle/event/energy_response_mean": 1.0039960145950317, "metrics/particle/event/met_abs_error": 4.041617393493652, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02425694279372692, "metrics/particle/event/scalar_pt_response_mean": 1.0057628154754639, "metrics/particle/event/vector_pt_closure": 0.026082666590809822, "metrics/particle/matched/delta_eta_abs_mean": 0.008427700027823448, "metrics/particle/matched/delta_phi_abs_mean": 0.00972186028957367, "metrics/particle/matched/delta_r_mean": 0.014380557462573051, "metrics/particle/matched/energy_relative_abs_mean": 0.08704786747694016, "metrics/particle/matched/pid_accuracy": 0.9769270420074463, "metrics/particle/matched/pt_relative_abs_mean": 0.08103140443563461, "metrics/particle/matching/duplicate_fraction": 0.025488529354333878, "metrics/particle/matching/efficiency": 0.8704171180725098, "metrics/particle/matching/f1": 0.8674312233924866, "metrics/particle/matching/purity": 0.8644657731056213, "metrics/particle/matching/target_coverage_dr0p05": 0.8501710891723633, "metrics/particle/matching/target_coverage_dr0p10": 0.8990548849105835, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7665390372276306, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9027619361877441}, "step_train_time": 0.8278236389160156, "step_valid_time": 2.0424691070220433, "step_total_time": 750.8890588283539, "elapsed_training_loop_seconds": 750.8890588283539, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028015582911678, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05855385778282618, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1500, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494344644045243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9513333333333334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9503829503829504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048666666666666664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9461077844311377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.948, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9470529470529471, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494344644045243, "step_test_time": 3.1791674430132844, "step_plot_time": 2.7678206879645586, "step_validation_cycle_time": 8.355560683994554} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..08000275c08b71585cd338a16ba4a8a66efd04ca --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..0f0822f99795bb3e86bf6829b0fe51e5b994d476 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17cd1c5e7eb04333a267b58e71d0bf1879b7c814176b04c9728b65284d791e7b +size 4498 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e9b6db25856fe5ba1271d81d2e8309f43cc5b462 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..e4b390d4b463c06a8b243ce8d56729a53245750e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a55c34a1f1a62c58cb0e2b985975b494a0c22b26 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..8b08d616a3d2a67c2876b08dd267329d7054d986 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ef8d93bd5d34c11bdabca8ecdc1341cfb8bdccec Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..640ba820870c77ecf1405bbf5e8736a418ea50f7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a346f1b426e026dd4074874e37d6e5010b51662c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..ccc110f1f73efc95c3611f6cd550f8d5d94f629c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..94d2c72a8bc7d5badee74ed6415a24d6254f6964 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..0bed44ce871842eda3e51f03a24ec411682a1ab5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b1df3143c79c4108cc60558ddf73aee4124b79a8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b1b913e8c4df96587e4bdb1de5afd4b9c3268d5a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/plots_step_33000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/train/events.out.tfevents.1794734505.workergpu204.2055039.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/train/events.out.tfevents.1794734505.workergpu204.2055039.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7eea3a2b4aa1de93407d47d4146cff8af22a2a4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/train/events.out.tfevents.1794734505.workergpu204.2055039.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a60d127008f2089589b764bb411d955049cb6dfc630827837c57a3aba1dcb9 +size 1214204 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/valid/events.out.tfevents.1794734505.workergpu204.2055039.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/valid/events.out.tfevents.1794734505.workergpu204.2055039.1 new file mode 100644 index 0000000000000000000000000000000000000000..a2da0306bfcea4cb400f3592f08caed1cddc212b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/runs/valid/events.out.tfevents.1794734505.workergpu204.2055039.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a8a34de000c6518fe188b5b19d4026e8ec644f4931921804fb00c205f4e893d +size 16817651 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..83c3a25e8927badbf704a368515371463a02c704 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_25mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e0a6fa74cfcc321f4b95f2f7d973aacf67fd57 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log new file mode 100644 index 0000000000000000000000000000000000000000..557404f7fbd9726383051ae963e8ee6320f60e06 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log @@ -0,0 +1,2630 @@ +rank0 [2026-11-15 04:21:44,005] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-15 04:21:44,005] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 04:21:44,261] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 04:21:44,261] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 04:21:44,261] INFO : Instantiating model +rank0 [2026-11-15 04:21:44,279] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 04:21:44,280] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 04:21:44,281] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 04:21:44,281] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 04:21:44,281] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 04:21:44,281] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 04:21:44,281] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,283] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 04:21:44,283] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,285] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 04:21:44,285] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,287] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 04:21:44,288] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,289] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 04:21:44,289] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,292] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 04:21:44,292] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,294] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 04:21:44,294] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 04:21:44,294] INFO : Initializing output DNNs +rank0 [2026-11-15 04:21:44,295] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,297] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,303] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 04:21:44,304] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 04:21:44,304] INFO : jagged_attention enabled=True +rank0 [2026-11-15 04:21:44,304] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 04:21:44,304] INFO : task_queries enabled=True +rank0 [2026-11-15 04:21:44,304] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 04:21:44,304] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 04:21:44,304] INFO : classification_query parameters: 256 +rank0 [2026-11-15 04:21:44,304] INFO : regression_query parameters: 256 +rank0 [2026-11-15 04:21:44,304] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 04:21:44,304] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 04:21:44,304] INFO : output_mode=elementwise +rank0 [2026-11-15 04:21:44,304] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 04:21:44,304] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 04:21:44,304] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 04:21:44,304] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 04:21:44,305] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 04:21:44,305] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 04:21:44,305] INFO : MLPF __init__ done +rank0 [2026-11-15 04:21:44,306] INFO : Instantiated model +rank0 [2026-11-15 04:21:44,306] INFO : Moving model to device rank=0 +rank0 [2026-11-15 04:21:44,312] INFO : Moved model to device rank=0 +rank0 [2026-11-15 04:21:44,313] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 04:21:44,388] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 04:21:44,388] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 04:21:44,388] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 04:21:44,448] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 04:21:44,448] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 04:21:44,449] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 04:21:44,449] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 04:21:44,449] INFO : Total parameters: 3910930 +rank0 [2026-11-15 04:21:44,451] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 04:21:44,452] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951 +rank0 [2026-11-15 04:21:44,452] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951 +rank0 [2026-11-15 04:21:44,452] INFO : Getting dataloaders +rank0 [2026-11-15 04:21:44,452] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:44,630] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,696] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,742] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,798] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,844] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,877] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,064] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,112] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,152] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,191] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,191] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 04:21:45,191] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 04:21:45,192] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 04:21:45,192] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:45,202] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,212] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,223] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,235] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,247] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,258] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,267] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,277] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,286] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,298] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,298] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 04:21:45,298] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 04:21:45,339] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:45,592] INFO : Got dataloaders +rank0 [2026-11-15 04:21:45,592] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 04:21:45,592] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 04:21:45,593] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 04:21:45,597] INFO : Creating train_iterator +rank0 [2026-11-15 04:21:45,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:45,634] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:45,806] INFO : Created train_iterator +rank0 [2026-11-15 04:21:54,553] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9476827490329742, 'Classification': 0.014934704909101128, 'Regression_pt': 0.0958048278465867, 'Regression_eta': 0.005884120861301198, 'Regression_sin_phi': 0.005008649970404804, 'Regression_cos_phi': 0.004921804838813841, 'Regression_energy': 0.09537884149700403}, weights={'Classification_binary': 1.0, 'Classification': 130.4132080078125, 'Regression_pt': 20.329692840576172, 'Regression_eta': 331.006591796875, 'Regression_sin_phi': 388.86383056640625, 'Regression_cos_phi': 395.7253112792969, 'Regression_energy': 20.420490264892578} +rank0 [2026-11-15 04:22:29,176] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 04:22:29,176] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:02,603] INFO : Step 1000/500000 rank0 | Train Loss: 14.8141 | Train Loss Unweighted: 2.2866 | Train Loss Components: Classification: 0.0148 | Classification_binary: 2.0414 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1069 | Regression_eta: 0.0037 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9354 | Classification_binary: 2.0414 | Regression_cos_phi: 2.0280 | Regression_energy: 2.1837 | Regression_eta: 1.2158 | Regression_pt: 2.1620 | Regression_sin_phi: 3.2478 | LR: 2.01e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7849s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0824 | max_tensorboard_logging: 0.2585 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2628.5 | uss_MB: 1357.2 | child_rss_MB: 23089.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:23:02,604] INFO : Running validation on rank0 +rank0 [2026-11-15 04:23:02,604] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:23:02,604] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:02,727] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 04:23:02,842] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:02,898] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:02,937] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:02,977] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:03,010] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:03,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:03,088] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:03,123] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:03,178] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:03,232] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:03,270] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:03,309] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:03,342] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:03,381] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:03,420] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:03,454] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:03,456] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:03,458] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:03,459] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:03,461] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:03,462] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:03,464] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:03,466] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:03,467] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:03,469] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:03,470] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:03,472] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:03,474] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:03,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:03,477] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:03,478] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:04,467] INFO : VALIDATION | Step=1000/500000 | Train Loss=14.8141 | Valid Loss=13.9802 | Train Loss Unweighted=2.2866 | Valid Loss Unweighted=2.1865 | Stale=0 | ETA=655.9m +rank0 [2026-11-15 04:23:04,467] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:04,597] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:23:04,605] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:04,651] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:05,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:05,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:05,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:05,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:05,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:05,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:06,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:06,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:06,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:06,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:06,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:06,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:07,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:07,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:07,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:07,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:07,677] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:23:07,685] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:08,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:08,392] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:09,347] INFO : Plotted jets +rank0 [2026-11-15 04:23:10,484] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:20,184] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 04:23:20,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:02,130] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 04:24:02,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:25,711] INFO : Step 2000/500000 rank0 | Train Loss: 14.5811 | Train Loss Unweighted: 2.4369 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.1935 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1060 | Regression_eta: 0.0058 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0741 | Classification_binary: 2.1935 | Regression_cos_phi: 2.3744 | Regression_energy: 2.1649 | Regression_eta: 1.9223 | Regression_pt: 2.1419 | Regression_sin_phi: 1.7100 | LR: 2.02e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0255 | max_loss: 0.0025 | max_backward: 0.0294 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2784.9 | uss_MB: 1529.1 | child_rss_MB: 23095.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:24:25,713] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 04:24:25,765] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:25,765] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:25,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:25,869] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 04:24:25,949] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:26,027] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:26,075] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:26,121] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:26,166] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:26,212] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:26,258] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:26,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:26,406] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:26,483] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:26,531] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:26,576] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:26,621] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:26,667] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:26,713] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:26,778] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:26,783] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:26,784] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:26,786] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:26,788] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:26,791] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:26,791] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:26,793] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:26,796] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:26,797] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:26,798] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:26,800] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:26,803] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:26,803] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:26,805] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:26,807] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:27,809] INFO : VALIDATION | Step=2000/500000 | Train Loss=14.5811 | Valid Loss=13.9228 | Train Loss Unweighted=2.4369 | Valid Loss Unweighted=2.1837 | Stale=0 | ETA=673.2m +rank0 [2026-11-15 04:24:27,810] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:28,233] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:28,238] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:28,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:28,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:29,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:29,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:29,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:29,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:29,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:29,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:30,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:30,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:30,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:30,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:30,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:30,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:31,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:31,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:31,277] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:24:31,284] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:31,665] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:31,918] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:33,275] INFO : Plotted jets +rank0 [2026-11-15 04:24:33,991] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:52,923] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 04:24:52,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:35,089] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 04:25:35,089] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:50,039] INFO : Step 3000/500000 rank0 | Train Loss: 10.8090 | Train Loss Unweighted: 2.1529 | Train Loss Components: Classification: 0.0144 | Classification_binary: 1.9620 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0827 | Regression_eta: 0.0026 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.8792 | Classification_binary: 1.9620 | Regression_cos_phi: 1.5883 | Regression_energy: 1.6897 | Regression_eta: 0.8446 | Regression_pt: 1.7138 | Regression_sin_phi: 1.1315 | LR: 2.05e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8515s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0027 | max_backward: 0.0271 | max_optimizer_step: 0.0849 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.5 | uss_MB: 1576.6 | child_rss_MB: 23097.1 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:25:50,040] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:50,040] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:50,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:50,144] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 04:25:50,227] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:50,304] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:50,355] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:50,402] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:50,448] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:50,496] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:50,543] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:50,616] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:50,699] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:50,778] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:50,826] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:50,872] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:50,919] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:50,968] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:51,015] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:51,082] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:51,086] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:51,088] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:51,089] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:51,091] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:51,092] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:51,094] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:51,095] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:51,097] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:51,098] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:51,100] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:51,101] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:51,103] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:51,105] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:51,106] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:51,108] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:52,172] INFO : VALIDATION | Step=3000/500000 | Train Loss=10.8090 | Valid Loss=13.9179 | Train Loss Unweighted=2.1529 | Valid Loss Unweighted=2.1914 | Stale=0 | ETA=680.8m +rank0 [2026-11-15 04:25:52,173] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:52,318] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:52,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:52,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:52,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:52,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:53,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:53,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:53,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:53,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:53,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:53,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:54,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:54,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:54,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:54,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:54,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:54,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:55,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:55,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:55,361] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:55,368] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:55,757] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:56,016] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:57,387] INFO : Plotted jets +rank0 [2026-11-15 04:25:58,100] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:26,395] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 04:26:26,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:08,623] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:27:08,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:14,299] INFO : Step 4000/500000 rank0 | Train Loss: 12.2415 | Train Loss Unweighted: 1.8120 | Train Loss Components: Classification: 0.0149 | Classification_binary: 1.6091 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0869 | Regression_eta: 0.0034 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9390 | Classification_binary: 1.6091 | Regression_cos_phi: 2.0009 | Regression_energy: 1.7736 | Regression_eta: 1.1361 | Regression_pt: 1.7810 | Regression_sin_phi: 2.0019 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7956s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0839 | max_tensorboard_logging: 0.2580 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.3 | uss_MB: 1608.5 | child_rss_MB: 23100.1 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:27:14,301] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:27:14,349] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:14,349] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:14,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:14,453] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:27:14,533] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:14,612] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:14,659] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:14,705] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:14,752] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:14,798] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:14,846] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:14,916] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:15,002] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:15,079] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:15,126] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:15,173] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:15,222] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:15,270] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:15,318] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:15,383] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:15,387] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:15,389] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:15,391] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:15,392] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:15,394] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:15,395] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:15,397] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:15,398] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:15,400] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:15,402] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:15,404] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:15,405] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:15,407] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:15,408] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:15,411] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:16,400] INFO : VALIDATION | Step=4000/500000 | Train Loss=12.2415 | Valid Loss=13.9565 | Train Loss Unweighted=1.8120 | Valid Loss Unweighted=2.1904 | Stale=1 | ETA=683.7m +rank0 [2026-11-15 04:27:16,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:16,540] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:16,558] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,607] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:17,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:17,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:17,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:17,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:17,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:17,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:18,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:18,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:18,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:18,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:18,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:18,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:19,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:19,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:19,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:19,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:19,601] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:27:19,609] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:19,999] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:20,260] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:21,170] INFO : Plotted jets +rank0 [2026-11-15 04:27:22,357] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:00,153] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:28:00,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:38,716] INFO : Step 5000/500000 rank0 | Train Loss: 14.3780 | Train Loss Unweighted: 2.2767 | Train Loss Components: Classification: 0.0121 | Classification_binary: 2.0676 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0880 | Regression_eta: 0.0084 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5726 | Classification_binary: 2.0676 | Regression_cos_phi: 2.1802 | Regression_energy: 1.7977 | Regression_eta: 2.7861 | Regression_pt: 1.8206 | Regression_sin_phi: 2.1532 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8288s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0025 | max_backward: 0.0276 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2893.4 | uss_MB: 1630.8 | child_rss_MB: 23096.4 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:28:38,717] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:38,717] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:38,717] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:38,824] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:28:38,901] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:38,979] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:39,026] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:39,071] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:39,116] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:39,162] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:39,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:39,277] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:39,354] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:39,431] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:39,478] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:39,523] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:39,568] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:39,615] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,662] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,727] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:39,731] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:39,733] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:39,735] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:39,736] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:39,738] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:39,739] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:39,741] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:39,742] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:39,744] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:39,746] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:39,747] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:39,749] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:39,750] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:39,752] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:39,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:40,702] INFO : VALIDATION | Step=5000/500000 | Train Loss=14.3780 | Valid Loss=13.9897 | Train Loss Unweighted=2.2767 | Valid Loss Unweighted=2.1945 | Stale=2 | ETA=684.9m +rank0 [2026-11-15 04:28:40,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:41,249] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:41,254] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:41,317] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:41,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:41,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:42,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:42,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:42,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:42,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:42,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:42,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:43,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:43,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:43,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:43,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:43,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:43,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:44,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:44,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:44,353] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:28:44,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,749] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:45,002] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:45,928] INFO : Plotted jets +rank0 [2026-11-15 04:28:47,095] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:51,713] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:28:51,713] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:33,272] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:29:33,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:02,488] INFO : Step 6000/500000 rank0 | Train Loss: 13.1978 | Train Loss Unweighted: 2.3508 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.1205 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1016 | Regression_eta: 0.0056 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0972 | Classification_binary: 2.1205 | Regression_cos_phi: 1.8161 | Regression_energy: 2.0740 | Regression_eta: 1.8679 | Regression_pt: 2.0179 | Regression_sin_phi: 1.2041 | LR: 2.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8132s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2583 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.3 | uss_MB: 1576.6 | child_rss_MB: 23100.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:02,490] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:30:02,540] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:02,541] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:02,541] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:02,659] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:30:02,740] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:02,819] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:02,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:02,913] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:02,958] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:03,005] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,051] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,122] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:03,201] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:03,279] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:03,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:03,372] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:03,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:03,462] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:03,508] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:03,573] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:03,577] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:03,579] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:03,580] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:03,582] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:03,583] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:03,585] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:03,587] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:03,588] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:03,590] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:03,591] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:03,593] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:03,594] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:03,596] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:03,597] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:03,599] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:04,566] INFO : VALIDATION | Step=6000/500000 | Train Loss=13.1978 | Valid Loss=14.0344 | Train Loss Unweighted=2.3508 | Valid Loss Unweighted=2.1991 | Stale=3 | ETA=684.7m +rank0 [2026-11-15 04:30:04,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:04,702] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:04,709] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:04,771] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:05,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:05,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:05,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:05,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:05,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:06,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:06,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:06,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:06,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:06,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:06,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:07,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:07,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:07,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:07,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:07,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:07,775] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:07,783] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:08,168] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:08,424] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:09,798] INFO : Plotted jets +rank0 [2026-11-15 04:30:10,516] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:24,517] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:30:24,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:06,476] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:31:06,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:26,389] INFO : Step 7000/500000 rank0 | Train Loss: 11.9085 | Train Loss Unweighted: 2.0755 | Train Loss Components: Classification: 0.0152 | Classification_binary: 1.8821 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0816 | Regression_eta: 0.0033 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9879 | Classification_binary: 1.8821 | Regression_cos_phi: 1.5839 | Regression_energy: 1.6662 | Regression_eta: 1.0917 | Regression_pt: 1.7120 | Regression_sin_phi: 1.9848 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8269s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0027 | max_backward: 0.0324 | max_optimizer_step: 0.0888 | max_tensorboard_logging: 0.2522 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2948.6 | uss_MB: 1685.9 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:31:26,391] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:26,391] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:26,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:26,500] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:31:26,580] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:26,659] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:26,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:26,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:26,805] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:26,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:26,899] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:26,970] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:27,051] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:27,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:27,184] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:27,230] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:27,277] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:27,325] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:27,372] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:27,440] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:27,444] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:27,446] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:27,448] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:27,449] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:27,451] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:27,452] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:27,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:27,455] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:27,458] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:27,458] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:27,460] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:27,461] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:27,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:27,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:27,467] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:28,443] INFO : VALIDATION | Step=7000/500000 | Train Loss=11.9085 | Valid Loss=14.0624 | Train Loss Unweighted=2.0755 | Valid Loss Unweighted=2.1997 | Stale=4 | ETA=684.2m +rank0 [2026-11-15 04:31:28,444] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:29,008] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:29,013] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:29,068] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:29,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:29,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:29,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:30,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:30,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:30,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:30,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:30,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:30,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:31,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:31,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:31,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:31,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:31,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:31,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:32,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:32,148] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:32,159] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:32,550] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:32,806] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:34,208] INFO : Plotted jets +rank0 [2026-11-15 04:31:34,933] INFO : Plotted jet ratio +rank0 [2026-11-15 04:31:58,670] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:31:58,671] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:40,629] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:32:40,629] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:51,262] INFO : Step 8000/500000 rank0 | Train Loss: 11.3475 | Train Loss Unweighted: 2.5754 | Train Loss Components: Classification: 0.0131 | Classification_binary: 2.3900 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0815 | Regression_eta: 0.0028 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7109 | Classification_binary: 2.3900 | Regression_cos_phi: 1.5285 | Regression_energy: 1.6634 | Regression_eta: 0.9242 | Regression_pt: 1.6328 | Regression_sin_phi: 1.4977 | LR: 2.34e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0274 | max_optimizer_step: 0.0840 | max_tensorboard_logging: 0.3040 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2968.7 | uss_MB: 1706.1 | child_rss_MB: 23102.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:32:51,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:32:51,315] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:51,315] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:51,315] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:51,420] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:32:51,501] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:51,580] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:51,629] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:51,674] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:51,719] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:51,766] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:51,812] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:51,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:51,960] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:52,037] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:52,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:52,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:52,176] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:52,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:52,267] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:52,333] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:52,338] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:52,339] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:52,341] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:52,342] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:52,344] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:52,346] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:52,347] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:52,349] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:52,351] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:52,353] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:52,354] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:52,356] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:52,357] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:52,359] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:52,362] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:53,338] INFO : VALIDATION | Step=8000/500000 | Train Loss=11.3475 | Valid Loss=14.1343 | Train Loss Unweighted=2.5754 | Valid Loss Unweighted=2.2088 | Stale=5 | ETA=684.4m +rank0 [2026-11-15 04:32:53,338] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:53,991] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:53,999] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:54,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:54,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:54,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:54,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:55,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:55,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:55,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:55,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:55,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:55,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:56,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:56,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:56,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:56,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:56,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:56,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:56,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:57,071] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:32:57,078] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:57,463] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:57,716] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:59,081] INFO : Plotted jets +rank0 [2026-11-15 04:32:59,817] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:31,525] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:33:31,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:13,084] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:34:13,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:14,440] INFO : Step 9000/500000 rank0 | Train Loss: 13.1962 | Train Loss Unweighted: 2.1812 | Train Loss Components: Classification: 0.0156 | Classification_binary: 1.9737 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0883 | Regression_eta: 0.0047 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0389 | Classification_binary: 1.9737 | Regression_cos_phi: 2.2095 | Regression_energy: 1.8040 | Regression_eta: 1.5592 | Regression_pt: 1.8001 | Regression_sin_phi: 1.8107 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8269s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2912.8 | uss_MB: 1650.3 | child_rss_MB: 23101.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:34:14,442] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:14,442] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:14,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:14,561] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:34:14,644] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:14,723] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:14,774] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:14,821] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:14,867] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:14,913] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:14,960] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:15,031] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:15,110] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:15,190] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:15,238] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:15,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:15,329] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:15,376] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:15,421] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:15,488] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:15,493] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:15,494] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:15,496] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:15,498] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:15,499] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:15,501] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:15,502] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:15,504] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:15,505] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:15,507] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:15,509] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:15,510] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:15,512] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:15,513] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:15,515] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:16,488] INFO : VALIDATION | Step=9000/500000 | Train Loss=13.1962 | Valid Loss=14.2057 | Train Loss Unweighted=2.1812 | Valid Loss Unweighted=2.2139 | Stale=6 | ETA=682.8m +rank0 [2026-11-15 04:34:16,489] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:16,626] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:16,635] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:16,691] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:17,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:17,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:17,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:17,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:17,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:17,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:18,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:18,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:18,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:18,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:18,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:18,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:19,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:19,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:19,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:19,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:19,665] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:19,673] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:20,063] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:20,328] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:21,262] INFO : Plotted jets +rank0 [2026-11-15 04:34:22,430] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:04,118] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:35:04,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:38,396] INFO : Step 10000/500000 rank0 | Train Loss: 16.4031 | Train Loss Unweighted: 2.4290 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.1766 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1068 | Regression_eta: 0.0096 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.2988 | Classification_binary: 2.1766 | Regression_cos_phi: 2.5845 | Regression_energy: 2.1800 | Regression_eta: 3.1889 | Regression_pt: 2.1796 | Regression_sin_phi: 1.7946 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0026 | max_backward: 0.0282 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2986 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3022.0 | uss_MB: 1759.4 | child_rss_MB: 23102.9 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:38,397] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:35:38,444] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:38,444] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:38,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:38,547] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:35:38,628] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:38,706] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:38,755] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:38,800] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:38,845] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:38,891] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:38,937] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:39,007] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:39,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:39,162] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:39,209] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:39,254] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:39,300] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:39,347] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:39,393] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:39,458] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:39,463] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:39,464] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:39,466] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:39,467] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:39,469] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:39,471] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:39,472] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:39,474] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:39,475] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:39,477] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:39,478] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:39,480] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:39,481] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:39,483] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:39,484] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:40,434] INFO : VALIDATION | Step=10000/500000 | Train Loss=16.4031 | Valid Loss=14.2347 | Train Loss Unweighted=2.4290 | Valid Loss Unweighted=2.2174 | Stale=7 | ETA=681.8m +rank0 [2026-11-15 04:35:40,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:40,823] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:35:40,832] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:40,881] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:41,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:41,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:41,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:41,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:42,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:42,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:42,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:42,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:42,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:43,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:43,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:43,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:43,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:43,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:43,970] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:35:43,977] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:44,369] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:44,638] INFO : Plotted number of elements +rank0 [2026-11-15 04:35:46,067] INFO : Plotted jets +rank0 [2026-11-15 04:35:46,802] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:55,923] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:35:55,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:38,450] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:36:38,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:03,542] INFO : Step 11000/500000 rank0 | Train Loss: 15.1416 | Train Loss Unweighted: 2.1507 | Train Loss Components: Classification: 0.0187 | Classification_binary: 1.9016 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1041 | Regression_eta: 0.0070 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.4422 | Classification_binary: 1.9016 | Regression_cos_phi: 1.9325 | Regression_energy: 2.1260 | Regression_eta: 2.3024 | Regression_pt: 2.2093 | Regression_sin_phi: 2.2276 | LR: 2.63e-05 | DataLoad Time: 0.0112s | Model Forward Time: 0.8326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0900 | max_tensorboard_logging: 0.2739 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3052.0 | uss_MB: 1789.4 | child_rss_MB: 23105.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 25.6 +rank0 [2026-11-15 04:37:03,543] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:03,543] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:03,543] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:03,648] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:37:03,727] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:03,806] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:03,854] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:03,899] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:03,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:03,991] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:04,038] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:04,108] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:04,185] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:04,263] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:04,311] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:04,357] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:04,403] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:04,450] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:04,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:04,563] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:04,568] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:04,569] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:04,571] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:04,573] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:04,574] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:04,576] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:04,577] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:04,579] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:04,580] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:04,582] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:04,583] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:04,585] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:04,586] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:04,588] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:04,592] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:05,540] INFO : VALIDATION | Step=11000/500000 | Train Loss=15.1416 | Valid Loss=14.2935 | Train Loss Unweighted=2.1507 | Valid Loss Unweighted=2.2165 | Stale=8 | ETA=681.6m +rank0 [2026-11-15 04:37:05,540] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:05,679] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:05,687] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:05,736] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:06,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:06,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:06,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:06,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:06,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:07,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:07,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:07,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:07,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:07,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:07,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:08,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:08,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:08,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:08,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:08,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:08,731] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:08,738] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:09,131] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:09,385] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:10,306] INFO : Plotted jets +rank0 [2026-11-15 04:37:11,478] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:29,813] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:37:29,814] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:12,143] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:38:12,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:27,747] INFO : Step 12000/500000 rank0 | Train Loss: 14.0491 | Train Loss Unweighted: 2.1502 | Train Loss Components: Classification: 0.0160 | Classification_binary: 1.9716 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0714 | Regression_eta: 0.0070 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0868 | Classification_binary: 1.9716 | Regression_cos_phi: 2.7559 | Regression_energy: 1.4581 | Regression_eta: 2.3095 | Regression_pt: 1.4660 | Regression_sin_phi: 2.0013 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0279 | max_loss: 0.0025 | max_backward: 0.0277 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3061.5 | uss_MB: 1798.8 | child_rss_MB: 23104.1 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:38:27,749] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:38:27,797] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:27,797] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:27,797] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:27,902] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:38:27,983] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:28,061] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:28,114] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:28,162] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:28,207] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:28,255] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:28,304] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:28,375] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:28,453] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:28,531] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:28,577] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:28,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:28,668] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:28,715] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:28,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:28,825] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:28,830] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:28,831] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:28,833] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:28,834] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:28,836] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:28,837] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:28,839] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:28,841] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:28,842] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:28,844] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:28,845] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:28,847] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:28,848] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:28,850] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:28,851] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:29,837] INFO : VALIDATION | Step=12000/500000 | Train Loss=14.0491 | Valid Loss=14.2685 | Train Loss Unweighted=2.1502 | Valid Loss Unweighted=2.2237 | Stale=9 | ETA=680.7m +rank0 [2026-11-15 04:38:29,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:30,298] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:30,306] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:30,356] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:30,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:31,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:31,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:31,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:31,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:31,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:31,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:31,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:32,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:32,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:32,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:32,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:32,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:32,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:33,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:33,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:33,411] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:33,418] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:33,809] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:34,062] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:34,981] INFO : Plotted jets +rank0 [2026-11-15 04:38:36,162] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:03,891] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:39:03,891] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:46,496] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:39:46,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:52,865] INFO : Step 13000/500000 rank0 | Train Loss: 13.9720 | Train Loss Unweighted: 2.4257 | Train Loss Components: Classification: 0.0157 | Classification_binary: 2.1901 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1019 | Regression_eta: 0.0068 | Regression_pt: 0.1029 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0505 | Classification_binary: 2.1901 | Regression_cos_phi: 2.1612 | Regression_energy: 2.0804 | Regression_eta: 2.2352 | Regression_pt: 2.0910 | Regression_sin_phi: 1.1635 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8698s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0026 | max_backward: 0.0278 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.3186 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3099.4 | uss_MB: 1836.8 | child_rss_MB: 23102.6 | allocated_MB: 170.9 | reserved_MB: 672.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:39:52,867] INFO : Running validation on rank0 +rank0 [2026-11-15 04:39:52,867] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:39:52,867] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:39:52,975] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:39:53,055] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:39:53,133] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:39:53,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:39:53,228] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:39:53,274] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:39:53,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:39:53,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:39:53,437] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:39:53,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:39:53,591] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:39:53,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:39:53,684] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:39:53,729] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:39:53,775] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:39:53,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:39:53,886] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:39:53,890] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:39:53,892] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:39:53,894] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:39:53,895] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:39:53,897] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:39:53,899] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:39:53,900] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:39:53,902] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:39:53,903] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:39:53,905] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:39:53,906] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:39:53,908] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:39:53,910] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:39:53,911] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:39:53,913] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:39:54,882] INFO : VALIDATION | Step=13000/500000 | Train Loss=13.9720 | Valid Loss=14.3628 | Train Loss Unweighted=2.4257 | Valid Loss Unweighted=2.2238 | Stale=10 | ETA=680.1m +rank0 [2026-11-15 04:39:54,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:39:55,006] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:39:55,015] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:55,072] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:39:55,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:39:55,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:39:55,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:39:56,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:39:56,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:39:56,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:39:56,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:39:56,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:39:56,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:39:57,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:39:57,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:39:57,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:39:57,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:39:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:39:57,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:39:58,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:39:58,160] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:39:58,167] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:58,559] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:58,813] INFO : Plotted number of elements +rank0 [2026-11-15 04:39:59,732] INFO : Plotted jets +rank0 [2026-11-15 04:40:00,919] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:37,878] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:40:37,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:16,960] INFO : Step 14000/500000 rank0 | Train Loss: 10.5674 | Train Loss Unweighted: 2.2669 | Train Loss Components: Classification: 0.0113 | Classification_binary: 2.0889 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0753 | Regression_eta: 0.0039 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.4753 | Classification_binary: 2.0889 | Regression_cos_phi: 1.6410 | Regression_energy: 1.5383 | Regression_eta: 1.2984 | Regression_pt: 1.6481 | Regression_sin_phi: 0.8774 | LR: 3.02e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.9189s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0025 | max_backward: 0.0279 | max_optimizer_step: 0.0923 | max_tensorboard_logging: 0.3469 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3125.6 | uss_MB: 1862.9 | child_rss_MB: 23104.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:16,962] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:41:17,012] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:17,012] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:17,012] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:17,119] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:41:17,201] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:17,280] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:17,329] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:17,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:17,428] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:17,475] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:17,522] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:17,593] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:17,673] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:17,751] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:17,799] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:17,845] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:17,890] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:17,937] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:17,983] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:18,050] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:18,055] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:18,056] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:18,058] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:18,059] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:18,061] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:18,063] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:18,064] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:18,066] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:18,067] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:18,069] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:18,070] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:18,073] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:18,073] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:18,075] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:18,076] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:19,064] INFO : VALIDATION | Step=14000/500000 | Train Loss=10.5674 | Valid Loss=14.4554 | Train Loss Unweighted=2.2669 | Valid Loss Unweighted=2.2346 | Stale=11 | ETA=678.9m +rank0 [2026-11-15 04:41:19,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:19,523] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:19,534] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:19,592] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:20,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:20,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:20,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:20,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:20,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:21,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:21,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:21,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:21,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:21,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:21,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:22,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:22,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:22,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:22,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:22,668] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:22,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:23,058] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:23,322] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:24,238] INFO : Plotted jets +rank0 [2026-11-15 04:41:25,435] INFO : Plotted jet ratio +rank0 [2026-11-15 04:41:29,595] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:41:29,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:11,481] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:42:11,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:40,967] INFO : Step 15000/500000 rank0 | Train Loss: 16.4500 | Train Loss Unweighted: 2.2700 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.0440 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0911 | Regression_eta: 0.0077 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.1864 | Classification_binary: 2.0440 | Regression_cos_phi: 2.6354 | Regression_energy: 1.8603 | Regression_eta: 2.5484 | Regression_pt: 1.9402 | Regression_sin_phi: 3.2353 | LR: 3.17e-05 | DataLoad Time: 0.0034s | Model Forward Time: 0.8666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0025 | max_backward: 0.0280 | max_optimizer_step: 0.0901 | max_tensorboard_logging: 0.3065 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3151.4 | uss_MB: 1888.7 | child_rss_MB: 23109.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:42:40,968] INFO : Running validation on rank0 +rank0 [2026-11-15 04:42:40,968] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:42:40,968] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:42:41,080] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:42:41,161] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:41,240] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:41,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:41,339] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:41,385] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:41,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:41,478] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:41,552] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:41,632] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:41,710] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:41,758] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:41,809] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:41,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:41,904] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:41,950] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:42,017] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:42,022] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:42,023] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:42,025] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:42,027] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:42,028] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:42,030] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:42,032] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:42,033] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:42,035] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:42,036] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:42,038] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:42,040] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:42,041] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:42,043] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:42,044] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:43,015] INFO : VALIDATION | Step=15000/500000 | Train Loss=16.4500 | Valid Loss=14.5445 | Train Loss Unweighted=2.2700 | Valid Loss Unweighted=2.2374 | Stale=12 | ETA=677.6m +rank0 [2026-11-15 04:42:43,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:43,154] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:43,181] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:43,242] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:43,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:43,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:44,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:44,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:44,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:44,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:44,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:44,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:45,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:45,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:45,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:45,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:45,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:45,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:46,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:46,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:46,362] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:42:46,369] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:46,765] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:47,023] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:47,991] INFO : Plotted jets +rank0 [2026-11-15 04:42:49,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:43:02,826] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:43:02,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:44,991] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:43:44,991] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:44:05,807] INFO : Step 16000/500000 rank0 | Train Loss: 9.9865 | Train Loss Unweighted: 2.1048 | Train Loss Components: Classification: 0.0148 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0715 | Regression_eta: 0.0024 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9280 | Classification_binary: 1.9383 | Regression_cos_phi: 1.0220 | Regression_energy: 1.4593 | Regression_eta: 0.7914 | Regression_pt: 1.4568 | Regression_sin_phi: 1.3908 | LR: 3.33e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8544s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0026 | max_backward: 0.0265 | max_optimizer_step: 0.0849 | max_tensorboard_logging: 0.2984 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3175.7 | uss_MB: 1913.1 | child_rss_MB: 23111.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:44:05,809] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:44:05,858] INFO : Running validation on rank0 +rank0 [2026-11-15 04:44:05,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:44:05,858] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:44:05,972] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:44:06,052] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:44:06,132] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:44:06,180] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:44:06,227] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:44:06,273] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:44:06,321] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:44:06,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:44:06,442] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:44:06,521] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:44:06,598] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:44:06,647] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:44:06,693] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:44:06,740] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:44:06,787] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:44:06,839] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:44:06,905] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:44:06,909] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:44:06,911] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:44:06,914] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:44:06,914] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:44:06,917] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:44:06,919] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:44:06,919] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:44:06,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:44:06,923] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:44:06,924] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:44:06,925] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:44:06,927] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:44:06,930] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:44:06,930] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:44:06,932] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:07,894] INFO : VALIDATION | Step=16000/500000 | Train Loss=9.9865 | Valid Loss=14.5846 | Train Loss Unweighted=2.1048 | Valid Loss Unweighted=2.2441 | Stale=13 | ETA=676.7m +rank0 [2026-11-15 04:44:07,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:08,314] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:08,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:08,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:08,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:09,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:09,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:09,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:09,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:09,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:09,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:10,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:10,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:10,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:10,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:10,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:10,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:11,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:11,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:11,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:11,450] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:44:11,457] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:11,831] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:12,084] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:13,028] INFO : Plotted jets +rank0 [2026-11-15 04:44:14,217] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:36,395] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:44:36,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:18,107] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:45:18,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:29,347] INFO : Step 17000/500000 rank0 | Train Loss: 13.2417 | Train Loss Unweighted: 2.1527 | Train Loss Components: Classification: 0.0107 | Classification_binary: 1.9703 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0772 | Regression_eta: 0.0063 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3977 | Classification_binary: 1.9703 | Regression_cos_phi: 1.9962 | Regression_energy: 1.5771 | Regression_eta: 2.0956 | Regression_pt: 1.5511 | Regression_sin_phi: 2.6538 | LR: 3.51e-05 | DataLoad Time: 0.0083s | Model Forward Time: 0.8214s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0025 | max_backward: 0.0278 | max_optimizer_step: 0.0901 | max_tensorboard_logging: 0.2545 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3111.9 | uss_MB: 1849.3 | child_rss_MB: 23105.8 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:45:29,349] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:29,349] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:29,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:29,460] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:45:29,543] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:29,623] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:29,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:29,719] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:29,765] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:29,811] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:29,858] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:29,933] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:30,011] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:30,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:30,138] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:30,184] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:30,230] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:30,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:30,327] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:30,394] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:30,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:30,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:30,402] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:30,404] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:30,405] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:30,407] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:30,408] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:30,410] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:30,411] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:30,413] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:30,414] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:30,416] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:30,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:30,419] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:30,421] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:31,387] INFO : VALIDATION | Step=17000/500000 | Train Loss=13.2417 | Valid Loss=14.7355 | Train Loss Unweighted=2.1527 | Valid Loss Unweighted=2.2478 | Stale=14 | ETA=675.2m +rank0 [2026-11-15 04:45:31,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:31,537] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:31,546] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:31,598] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:32,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:32,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:32,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:32,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:32,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:32,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:33,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:33,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:33,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:33,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:33,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:33,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:34,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:34,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:34,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:34,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:34,623] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:45:34,631] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:35,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:35,265] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:36,189] INFO : Plotted jets +rank0 [2026-11-15 04:45:37,381] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:09,755] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:46:09,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:52,101] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:46:52,101] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:54,073] INFO : Step 18000/500000 rank0 | Train Loss: 13.2638 | Train Loss Unweighted: 2.1536 | Train Loss Components: Classification: 0.0151 | Classification_binary: 1.9378 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0916 | Regression_eta: 0.0037 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9729 | Classification_binary: 1.9378 | Regression_cos_phi: 2.1334 | Regression_energy: 1.8714 | Regression_eta: 1.2239 | Regression_pt: 1.9171 | Regression_sin_phi: 2.2074 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8364s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0256 | max_loss: 0.0040 | max_backward: 0.0282 | max_optimizer_step: 0.0902 | max_tensorboard_logging: 0.2684 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3224.3 | uss_MB: 1961.7 | child_rss_MB: 23107.5 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:46:54,074] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:46:54,123] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:54,123] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:54,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:54,231] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:46:54,318] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:54,398] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:54,449] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:54,495] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:54,545] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:54,593] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:54,639] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:54,711] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:54,790] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:54,867] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:54,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:54,962] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:55,009] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:55,058] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:55,107] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:55,174] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:55,179] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:55,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:55,182] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:55,185] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:55,185] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:55,187] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:55,188] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:55,190] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:55,191] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:55,193] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:55,194] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:55,196] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:55,197] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:55,199] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:55,201] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:46:56,177] INFO : VALIDATION | Step=18000/500000 | Train Loss=13.2638 | Valid Loss=14.6714 | Train Loss Unweighted=2.1536 | Valid Loss Unweighted=2.2536 | Stale=15 | ETA=674.2m +rank0 [2026-11-15 04:46:56,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:46:56,311] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:46:56,319] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:56,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:56,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:57,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:57,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:57,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:57,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:57,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:58,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:58,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:58,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:58,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:58,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:58,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:59,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:59,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:59,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:59,464] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:46:59,483] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:59,879] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:00,141] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:01,084] INFO : Plotted jets +rank0 [2026-11-15 04:47:02,301] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:43,336] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:47:43,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:18,231] INFO : Step 19000/500000 rank0 | Train Loss: 11.1812 | Train Loss Unweighted: 2.2649 | Train Loss Components: Classification: 0.0147 | Classification_binary: 2.0556 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0928 | Regression_eta: 0.0030 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9204 | Classification_binary: 2.0556 | Regression_cos_phi: 1.1161 | Regression_energy: 1.8944 | Regression_eta: 0.9884 | Regression_pt: 1.8813 | Regression_sin_phi: 1.3250 | LR: 3.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2542 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3250.3 | uss_MB: 1987.7 | child_rss_MB: 23104.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:48:18,232] INFO : Running validation on rank0 +rank0 [2026-11-15 04:48:18,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:48:18,232] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:48:18,339] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:48:18,420] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:48:18,499] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:48:18,550] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:48:18,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:48:18,643] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:48:18,690] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:48:18,736] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:48:18,808] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:48:18,886] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:48:18,964] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:48:19,013] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:48:19,059] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:48:19,106] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:48:19,152] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:48:19,198] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:48:19,265] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:48:19,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:48:19,271] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:48:19,273] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:48:19,274] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:48:19,276] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:48:19,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:48:19,279] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:48:19,280] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:48:19,282] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:48:19,284] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:48:19,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:48:19,287] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:48:19,288] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:48:19,290] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:48:19,292] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:48:20,266] INFO : VALIDATION | Step=19000/500000 | Train Loss=11.1812 | Valid Loss=14.8614 | Train Loss Unweighted=2.2649 | Valid Loss Unweighted=2.2595 | Stale=16 | ETA=672.8m +rank0 [2026-11-15 04:48:20,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:48:20,428] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:48:20,436] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:20,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:48:21,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:48:21,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:48:21,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:48:21,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:48:21,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:48:21,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:48:22,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:22,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:22,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:22,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:22,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:22,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:23,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:23,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:23,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:23,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:23,633] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:48:23,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:24,016] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:24,269] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:25,201] INFO : Plotted jets +rank0 [2026-11-15 04:48:26,381] INFO : Plotted jet ratio +rank0 [2026-11-15 04:48:34,928] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:48:34,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:17,279] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:49:17,279] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:42,857] INFO : Step 20000/500000 rank0 | Train Loss: 13.5737 | Train Loss Unweighted: 1.9784 | Train Loss Components: Classification: 0.0134 | Classification_binary: 1.7792 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0806 | Regression_eta: 0.0067 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7436 | Classification_binary: 1.7792 | Regression_cos_phi: 2.2135 | Regression_energy: 1.6465 | Regression_eta: 2.2340 | Regression_pt: 1.7755 | Regression_sin_phi: 2.1815 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8719s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0027 | max_backward: 0.0285 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.3008 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3186.8 | uss_MB: 1924.2 | child_rss_MB: 23106.3 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 27.6 +rank0 [2026-11-15 04:49:42,859] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:49:42,921] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:42,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:42,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:43,025] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:49:43,109] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:43,187] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:43,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:43,282] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:43,328] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:43,374] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:43,421] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:43,492] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:43,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:43,649] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:43,697] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:43,745] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:43,797] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:43,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:43,891] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:43,962] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:43,967] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:43,969] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:43,970] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:43,973] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:43,973] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:43,975] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:43,978] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:43,978] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:43,981] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:43,982] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:43,983] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:43,985] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:43,986] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:43,988] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:43,989] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:44,950] INFO : VALIDATION | Step=20000/500000 | Train Loss=13.5737 | Valid Loss=14.7951 | Train Loss Unweighted=1.9784 | Valid Loss Unweighted=2.2705 | Stale=17 | ETA=671.7m +rank0 [2026-11-15 04:49:44,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:45,364] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:45,380] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:45,448] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:46,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:46,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:46,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:46,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:46,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:46,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:47,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:47,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:47,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:47,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:47,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:49:47,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:49:48,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:49:48,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:49:48,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:49:48,520] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:49:48,528] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:48,918] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,172] INFO : Plotted number of elements +rank0 [2026-11-15 04:49:50,100] INFO : Plotted jets +rank0 [2026-11-15 04:49:51,300] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:09,143] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:50:09,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:50:51,099] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:50:51,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:07,149] INFO : Step 21000/500000 rank0 | Train Loss: 10.2156 | Train Loss Unweighted: 1.8802 | Train Loss Components: Classification: 0.0132 | Classification_binary: 1.7134 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0716 | Regression_eta: 0.0035 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7206 | Classification_binary: 1.7134 | Regression_cos_phi: 1.6729 | Regression_energy: 1.4625 | Regression_eta: 1.1601 | Regression_pt: 1.4561 | Regression_sin_phi: 1.0301 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0275 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3297.0 | uss_MB: 2034.4 | child_rss_MB: 23104.3 | allocated_MB: 170.4 | reserved_MB: 712.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:51:07,150] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:07,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:07,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:07,256] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:51:07,342] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:07,421] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:07,471] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:07,519] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:07,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:07,613] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:07,660] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:07,735] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:07,816] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:07,894] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:07,942] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:07,992] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:08,039] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:08,086] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:08,139] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:08,209] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:08,214] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:08,216] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:08,217] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:08,219] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:08,221] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:08,222] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:08,225] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:08,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:08,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:08,230] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:08,230] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:08,232] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:08,235] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:08,235] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:08,237] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:09,241] INFO : VALIDATION | Step=21000/500000 | Train Loss=10.2156 | Valid Loss=14.9642 | Train Loss Unweighted=1.8802 | Valid Loss Unweighted=2.2708 | Stale=18 | ETA=670.5m +rank0 [2026-11-15 04:51:09,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:09,563] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:09,568] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:09,632] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:10,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:10,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:10,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:10,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:10,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:10,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:11,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:11,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:11,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:11,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:11,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:11,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:12,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:12,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:12,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:12,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:12,731] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:51:12,740] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:13,116] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:13,371] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:14,321] INFO : Plotted jets +rank0 [2026-11-15 04:51:15,497] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:42,422] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:51:42,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:24,415] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:52:24,415] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:31,434] INFO : Step 22000/500000 rank0 | Train Loss: 14.4647 | Train Loss Unweighted: 2.5983 | Train Loss Components: Classification: 0.0125 | Classification_binary: 2.3248 | Regression_cos_phi: 0.0030 | Regression_energy: 0.1300 | Regression_eta: 0.0089 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.6298 | Classification_binary: 2.3248 | Regression_cos_phi: 1.2046 | Regression_energy: 2.6555 | Regression_eta: 2.9478 | Regression_pt: 2.3490 | Regression_sin_phi: 1.3532 | LR: 4.50e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8921s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0274 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3317.8 | uss_MB: 2055.2 | child_rss_MB: 23106.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:52:31,436] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:52:31,487] INFO : Running validation on rank0 +rank0 [2026-11-15 04:52:31,487] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:52:31,487] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:52:31,591] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:52:31,671] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:52:31,750] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:52:31,799] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:52:31,846] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:52:31,891] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:52:31,937] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:52:31,984] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:52:32,055] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:52:32,134] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:52:32,211] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:52:32,259] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:52:32,305] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:52:32,352] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:52:32,398] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:52:32,444] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:52:32,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:52:32,516] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:52:32,518] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:52:32,519] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:52:32,521] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:52:32,523] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:52:32,525] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:52:32,526] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:52:32,528] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:52:32,531] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:52:32,531] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:52:32,533] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:52:32,535] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:52:32,537] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:52:32,538] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:52:32,540] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:52:33,505] INFO : VALIDATION | Step=22000/500000 | Train Loss=14.4647 | Valid Loss=15.0412 | Train Loss Unweighted=2.5983 | Valid Loss Unweighted=2.2767 | Stale=19 | ETA=669.2m +rank0 [2026-11-15 04:52:33,506] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:52:33,650] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:52:33,655] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:33,706] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:52:34,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:52:34,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:52:34,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:52:34,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:52:34,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:52:35,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:52:35,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:52:35,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:52:35,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:52:35,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:52:35,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:52:36,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:52:36,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:52:36,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:52:36,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:52:36,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:52:36,759] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:52:36,767] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:37,161] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:37,421] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:38,349] INFO : Plotted jets +rank0 [2026-11-15 04:52:39,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:53:15,954] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:53:15,954] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:55,769] INFO : Step 23000/500000 rank0 | Train Loss: 12.2439 | Train Loss Unweighted: 2.3527 | Train Loss Components: Classification: 0.0121 | Classification_binary: 2.1547 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0866 | Regression_eta: 0.0052 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5834 | Classification_binary: 2.1547 | Regression_cos_phi: 1.7519 | Regression_energy: 1.7688 | Regression_eta: 1.7169 | Regression_pt: 1.7407 | Regression_sin_phi: 1.5275 | LR: 4.73e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8425s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3340.8 | uss_MB: 2078.2 | child_rss_MB: 23107.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:53:55,770] INFO : Running validation on rank0 +rank0 [2026-11-15 04:53:55,771] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:53:55,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:53:55,877] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:53:55,958] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:53:56,038] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:53:56,088] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:53:56,134] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:53:56,180] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:53:56,228] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:53:56,275] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:53:56,346] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:53:56,425] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:53:56,504] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:53:56,552] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:53:56,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:53:56,654] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:53:56,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:53:56,747] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:53:56,813] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:53:56,817] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:53:56,820] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:53:56,821] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:53:56,822] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:53:56,824] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:53:56,825] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:53:56,827] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:53:56,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:53:56,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:53:56,832] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:53:56,834] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:53:56,835] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:53:56,837] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:53:56,838] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:53:56,840] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:53:57,792] INFO : VALIDATION | Step=23000/500000 | Train Loss=12.2439 | Valid Loss=15.1833 | Train Loss Unweighted=2.3527 | Valid Loss Unweighted=2.2832 | Stale=20 | ETA=667.9m +rank0 [2026-11-15 04:53:57,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:53:57,960] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:53:57,965] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:58,029] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:53:58,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:53:58,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:53:58,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:53:59,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:53:59,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:53:59,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:53:59,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:53:59,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:53:59,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:54:00,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:54:00,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:54:00,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:54:00,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:54:00,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:54:00,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:54:01,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:54:01,129] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:54:01,136] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:54:01,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:54:01,773] INFO : Plotted number of elements +rank0 [2026-11-15 04:54:03,175] INFO : Plotted jets +rank0 [2026-11-15 04:54:03,898] INFO : Plotted jet ratio +rank0 [2026-11-15 04:54:07,497] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:54:07,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:54:50,149] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:54:50,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:20,515] INFO : Step 24000/500000 rank0 | Train Loss: 8.9178 | Train Loss Unweighted: 1.9522 | Train Loss Components: Classification: 0.0126 | Classification_binary: 1.7864 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0742 | Regression_eta: 0.0017 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.6418 | Classification_binary: 1.7864 | Regression_cos_phi: 0.9834 | Regression_energy: 1.5159 | Regression_eta: 0.5472 | Regression_pt: 1.4711 | Regression_sin_phi: 0.9721 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8210s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0256 | max_loss: 0.0025 | max_backward: 0.0277 | max_optimizer_step: 0.0897 | max_tensorboard_logging: 0.2655 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3369.3 | uss_MB: 2106.7 | child_rss_MB: 23108.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:55:20,517] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:55:20,567] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:20,567] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:20,567] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:20,673] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:55:20,753] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:20,832] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:20,881] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:20,927] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:20,974] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:21,020] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:21,066] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:21,138] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:21,218] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:21,296] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:21,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:21,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:21,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:21,485] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:21,531] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:21,599] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:21,603] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:21,604] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:21,607] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:21,608] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:21,609] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:21,611] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:21,613] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:21,614] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:21,616] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:21,617] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:21,619] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:21,620] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:21,622] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:21,624] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:21,625] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:22,593] INFO : VALIDATION | Step=24000/500000 | Train Loss=8.9178 | Valid Loss=15.2309 | Train Loss Unweighted=1.9522 | Valid Loss Unweighted=2.2872 | Stale=21 | ETA=666.7m +rank0 [2026-11-15 04:55:22,593] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:55:22,938] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:55:22,943] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:23,006] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:23,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:23,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:23,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:24,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:24,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:24,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:24,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:24,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:24,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:25,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:25,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:25,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:25,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:25,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:25,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:26,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:26,114] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:55:26,122] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:26,504] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:26,760] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:27,688] INFO : Plotted jets +rank0 [2026-11-15 04:55:28,884] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:41,477] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:55:41,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:22,988] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:56:22,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:44,055] INFO : Step 25000/500000 rank0 | Train Loss: 10.4613 | Train Loss Unweighted: 1.9660 | Train Loss Components: Classification: 0.0100 | Classification_binary: 1.8029 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0693 | Regression_eta: 0.0039 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3040 | Classification_binary: 1.8029 | Regression_cos_phi: 1.6712 | Regression_energy: 1.4145 | Regression_eta: 1.3008 | Regression_pt: 1.4603 | Regression_sin_phi: 1.5078 | LR: 5.22e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0253 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2960 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3394.3 | uss_MB: 2131.7 | child_rss_MB: 23107.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:56:44,056] INFO : Running validation on rank0 +rank0 [2026-11-15 04:56:44,056] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:56:44,056] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:56:44,161] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:56:44,242] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:56:44,321] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:56:44,370] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:56:44,417] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:56:44,464] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:56:44,512] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:56:44,559] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:56:44,631] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:56:44,709] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:56:44,789] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:56:44,837] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:56:44,882] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:56:44,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:56:44,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:56:45,022] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:56:45,088] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:56:45,092] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:56:45,094] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:56:45,096] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:56:45,101] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:56:45,103] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:56:45,104] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:56:45,106] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:56:45,107] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:56:45,109] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:56:45,111] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:56:45,112] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:56:45,114] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:56:45,115] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:56:45,117] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:56:45,119] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:56:46,064] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.4613 | Valid Loss=15.3322 | Train Loss Unweighted=1.9660 | Valid Loss Unweighted=2.2940 | Stale=22 | ETA=665.1m +rank0 [2026-11-15 04:56:46,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:56:46,412] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:56:46,417] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:46,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:56:47,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:56:47,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:56:47,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:56:47,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:56:47,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:56:47,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:56:47,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:56:48,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:56:48,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:56:48,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:56:48,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:56:48,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:56:48,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:56:49,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:56:49,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:56:49,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:56:49,547] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:56:49,555] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:49,943] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:50,199] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:51,126] INFO : Plotted jets +rank0 [2026-11-15 04:56:52,310] INFO : Plotted jet ratio +rank0 [2026-11-15 04:57:14,594] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:57:14,594] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:57:57,342] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:57:57,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:58:09,338] INFO : Step 26000/500000 rank0 | Train Loss: 8.3759 | Train Loss Unweighted: 2.0088 | Train Loss Components: Classification: 0.0122 | Classification_binary: 1.8517 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0693 | Regression_eta: 0.0018 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5858 | Classification_binary: 1.8517 | Regression_cos_phi: 0.9338 | Regression_energy: 1.4142 | Regression_eta: 0.6049 | Regression_pt: 1.4259 | Regression_sin_phi: 0.5596 | LR: 5.47e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3420.3 | uss_MB: 2157.8 | child_rss_MB: 23110.0 | allocated_MB: 170.4 | reserved_MB: 666.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:58:09,340] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:58:09,390] INFO : Running validation on rank0 +rank0 [2026-11-15 04:58:09,390] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:58:09,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:58:09,503] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:58:09,583] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:58:09,662] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:58:09,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:58:09,758] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:58:09,804] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:58:09,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:58:09,898] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:58:09,969] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:58:10,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:58:10,128] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:58:10,176] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:58:10,223] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:58:10,267] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:58:10,314] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:58:10,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:58:10,428] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:58:10,432] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:58:10,434] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:58:10,436] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:58:10,437] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:58:10,439] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:58:10,440] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:58:10,442] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:58:10,443] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:58:10,445] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:58:10,447] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:58:10,448] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:58:10,450] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:58:10,451] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:58:10,453] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:58:10,454] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:58:11,426] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.3759 | Valid Loss=15.2828 | Train Loss Unweighted=2.0088 | Valid Loss Unweighted=2.2940 | Stale=23 | ETA=664.2m +rank0 [2026-11-15 04:58:11,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:58:11,548] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:58:11,554] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:11,616] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:58:12,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:58:12,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:58:12,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:58:12,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:58:12,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:58:12,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:58:13,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:58:13,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:58:13,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:58:13,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:58:13,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:58:13,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:58:14,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:58:14,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:58:14,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:58:14,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:58:14,667] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:58:14,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:15,063] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:15,320] INFO : Plotted number of elements +rank0 [2026-11-15 04:58:16,254] INFO : Plotted jets +rank0 [2026-11-15 04:58:17,448] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:48,351] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:58:48,351] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:30,350] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:59:30,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:32,912] INFO : Step 27000/500000 rank0 | Train Loss: 10.3843 | Train Loss Unweighted: 1.9298 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.7659 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0697 | Regression_eta: 0.0046 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3860 | Classification_binary: 1.7659 | Regression_cos_phi: 1.0365 | Regression_energy: 1.4233 | Regression_eta: 1.5109 | Regression_pt: 1.4586 | Regression_sin_phi: 1.8032 | LR: 5.74e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8398s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0024 | max_backward: 0.0275 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3450.5 | uss_MB: 2187.9 | child_rss_MB: 23108.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:59:32,914] INFO : Running validation on rank0 +rank0 [2026-11-15 04:59:32,914] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:59:32,914] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:59:33,017] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:59:33,097] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:59:33,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:59:33,229] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:59:33,277] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:59:33,324] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:59:33,371] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:59:33,417] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:59:33,489] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:59:33,569] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:59:33,649] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:59:33,697] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:59:33,743] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:59:33,789] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:59:33,836] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:59:33,882] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:59:33,953] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:59:33,958] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:59:33,961] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:59:33,962] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:59:33,963] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:59:33,964] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:59:33,966] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:59:33,968] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:59:33,970] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:59:33,972] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:59:33,973] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:59:33,974] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:59:33,976] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:59:33,977] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:59:33,979] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:59:33,980] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:59:34,950] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.3843 | Valid Loss=15.6013 | Train Loss Unweighted=1.9298 | Valid Loss Unweighted=2.3034 | Stale=24 | ETA=662.6m +rank0 [2026-11-15 04:59:34,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:59:35,079] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:59:35,084] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:35,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:59:35,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:59:35,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:59:36,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:59:36,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:59:36,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:59:36,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:59:36,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:59:36,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:59:37,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:59:37,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:59:37,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:59:37,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:59:37,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:59:37,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:59:38,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:59:38,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:59:38,239] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:59:38,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:38,625] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:38,881] INFO : Plotted number of elements +rank0 [2026-11-15 04:59:39,812] INFO : Plotted jets +rank0 [2026-11-15 04:59:41,013] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:22,196] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 05:00:22,197] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:57,854] INFO : Step 28000/500000 rank0 | Train Loss: 9.0165 | Train Loss Unweighted: 2.1042 | Train Loss Components: Classification: 0.0115 | Classification_binary: 1.9649 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0590 | Regression_eta: 0.0025 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5024 | Classification_binary: 1.9649 | Regression_cos_phi: 1.4890 | Regression_energy: 1.2046 | Regression_eta: 0.8243 | Regression_pt: 1.2286 | Regression_sin_phi: 0.8027 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0025 | max_backward: 0.0265 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2509 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3470.6 | uss_MB: 2208.0 | child_rss_MB: 23110.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:00:57,856] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 05:00:57,907] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:57,907] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:57,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:58,012] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 05:00:58,094] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:58,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:58,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:58,272] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:58,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:58,365] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:58,413] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:58,486] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:58,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:58,650] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:58,699] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:58,746] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:58,793] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:58,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:58,890] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:58,958] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:58,963] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:58,964] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:58,966] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:58,968] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:58,969] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:58,971] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:58,972] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:58,974] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:58,975] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:58,977] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:58,979] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:58,980] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:58,982] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:58,983] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:58,985] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:59,955] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.0165 | Valid Loss=15.5100 | Train Loss Unweighted=2.1042 | Valid Loss Unweighted=2.3018 | Stale=25 | ETA=661.5m +rank0 [2026-11-15 05:00:59,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:01:00,295] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:01:00,300] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:00,364] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:01:00,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:01:01,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:01:01,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:01:01,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:01:01,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:01:01,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:01:01,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:01:02,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:01:02,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:01:02,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:01:02,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:01:02,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:01:02,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:01:03,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:01:03,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:01:03,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:01:03,522] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:01:03,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:03,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:04,189] INFO : Plotted number of elements +rank0 [2026-11-15 05:01:05,172] INFO : Plotted jets +rank0 [2026-11-15 05:01:06,386] INFO : Plotted jet ratio +rank0 [2026-11-15 05:01:14,174] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 05:01:14,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:01:55,837] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 05:01:55,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:02:21,617] INFO : Step 29000/500000 rank0 | Train Loss: 10.4576 | Train Loss Unweighted: 1.7359 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.5747 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0682 | Regression_eta: 0.0057 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.4260 | Classification_binary: 1.5747 | Regression_cos_phi: 1.2835 | Regression_energy: 1.3927 | Regression_eta: 1.8902 | Regression_pt: 1.4083 | Regression_sin_phi: 1.4823 | LR: 6.29e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0283 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2681 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3492.5 | uss_MB: 2230.0 | child_rss_MB: 23108.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:02:21,619] INFO : Running validation on rank0 +rank0 [2026-11-15 05:02:21,619] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:02:21,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:02:21,724] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 05:02:21,803] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:02:21,883] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:02:21,933] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:02:21,980] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:02:22,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:02:22,074] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:02:22,121] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:02:22,194] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:02:22,273] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:02:22,351] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:02:22,400] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:02:22,446] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:02:22,492] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:02:22,538] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:02:22,584] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:02:22,652] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:02:22,657] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:02:22,659] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:02:22,660] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:02:22,662] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:02:22,664] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:02:22,665] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:02:22,667] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:02:22,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:02:22,670] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:02:22,672] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:02:22,673] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:02:22,675] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:02:22,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:02:22,678] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:02:22,680] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:02:23,629] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.4576 | Valid Loss=15.5704 | Train Loss Unweighted=1.7359 | Valid Loss Unweighted=2.3133 | Stale=26 | ETA=660.0m +rank0 [2026-11-15 05:02:23,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:02:23,781] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:02:23,786] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:23,837] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:02:24,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:02:24,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:02:24,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:02:24,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:02:24,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:02:25,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:02:25,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:02:25,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:02:25,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:02:25,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:02:25,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:02:26,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:02:26,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:02:26,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:02:26,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:02:26,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:02:26,877] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:02:26,884] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:27,275] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:27,529] INFO : Plotted number of elements +rank0 [2026-11-15 05:02:28,492] INFO : Plotted jets +rank0 [2026-11-15 05:02:29,697] INFO : Plotted jet ratio +rank0 [2026-11-15 05:02:47,186] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 05:02:47,186] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:29,810] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 05:03:29,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:46,740] INFO : Step 30000/500000 rank0 | Train Loss: 9.5773 | Train Loss Unweighted: 2.0052 | Train Loss Components: Classification: 0.0118 | Classification_binary: 1.8595 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0615 | Regression_eta: 0.0052 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5433 | Classification_binary: 1.8595 | Regression_cos_phi: 1.1315 | Regression_energy: 1.2561 | Regression_eta: 1.7376 | Regression_pt: 1.2654 | Regression_sin_phi: 0.7840 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8262s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0026 | max_backward: 0.0275 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2671 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3512.7 | uss_MB: 2250.2 | child_rss_MB: 23108.2 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:03:46,742] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 05:03:46,791] INFO : Running validation on rank0 +rank0 [2026-11-15 05:03:46,791] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:03:46,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:03:46,919] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 05:03:47,003] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:03:47,083] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:03:47,133] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:03:47,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:03:47,225] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:03:47,272] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:03:47,319] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:03:47,390] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:03:47,470] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:03:47,547] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:03:47,595] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:03:47,642] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:03:47,688] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:03:47,734] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:03:47,780] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:03:47,847] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:03:47,852] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:03:47,854] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:03:47,855] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:03:47,857] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:03:47,859] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:03:47,860] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:03:47,862] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:03:47,863] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:03:47,865] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:03:47,866] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:03:47,868] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:03:47,870] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:03:47,871] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:03:47,873] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:03:47,874] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:03:48,841] INFO : VALIDATION | Step=30000/500000 | Train Loss=9.5773 | Valid Loss=15.8482 | Train Loss Unweighted=2.0052 | Valid Loss Unweighted=2.3270 | Stale=27 | ETA=658.8m +rank0 [2026-11-15 05:03:48,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:03:48,965] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:03:48,970] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:49,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:03:49,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:03:49,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:03:49,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:03:50,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:03:50,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:03:50,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:03:50,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:03:50,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:03:50,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:03:51,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:03:51,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:03:51,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:03:51,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:03:51,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:03:51,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:03:52,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:03:52,104] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:03:52,112] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:52,499] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:52,754] INFO : Plotted number of elements +rank0 [2026-11-15 05:03:53,679] INFO : Plotted jets +rank0 [2026-11-15 05:03:54,893] INFO : Plotted jet ratio +rank0 [2026-11-15 05:04:21,570] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 05:04:21,570] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:05:04,054] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 05:05:04,055] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:05:11,638] INFO : Step 31000/500000 rank0 | Train Loss: 12.6850 | Train Loss Unweighted: 2.1436 | Train Loss Components: Classification: 0.0151 | Classification_binary: 1.9460 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0836 | Regression_eta: 0.0081 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9664 | Classification_binary: 1.9460 | Regression_cos_phi: 1.2331 | Regression_energy: 1.7075 | Regression_eta: 2.6865 | Regression_pt: 1.7071 | Regression_sin_phi: 1.4385 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8818s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.3208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3458.8 | uss_MB: 2196.2 | child_rss_MB: 23102.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 05:05:11,640] INFO : Running validation on rank0 +rank0 [2026-11-15 05:05:11,640] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:05:11,640] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:05:11,745] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 05:05:11,826] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:05:11,905] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:05:11,956] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:05:12,003] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:05:12,051] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:05:12,099] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:05:12,146] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:05:12,218] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:05:12,306] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:05:12,384] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:05:12,432] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:05:12,481] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:05:12,528] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:05:12,576] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:05:12,625] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:05:12,695] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:05:12,699] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:05:12,701] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:05:12,703] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:05:12,705] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:05:12,706] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:05:12,709] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:05:12,709] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:05:12,711] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:05:12,714] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:05:12,714] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:05:12,717] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:05:12,717] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:05:12,719] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:05:12,721] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:05:12,722] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:05:13,717] INFO : VALIDATION | Step=31000/500000 | Train Loss=12.6850 | Valid Loss=15.8210 | Train Loss Unweighted=2.1436 | Valid Loss Unweighted=2.3241 | Stale=28 | ETA=657.6m +rank0 [2026-11-15 05:05:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:05:14,053] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:05:14,058] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:14,110] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:05:14,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:05:14,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:05:14,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:05:15,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:05:15,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:05:15,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:05:15,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:15,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:15,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:16,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:16,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:16,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:16,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:16,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:16,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:17,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:17,209] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:05:17,216] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,623] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,882] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:18,825] INFO : Plotted jets +rank0 [2026-11-15 05:05:20,108] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:56,054] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 05:05:56,054] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:36,900] INFO : Step 32000/500000 rank0 | Train Loss: 12.2766 | Train Loss Unweighted: 2.4380 | Train Loss Components: Classification: 0.0118 | Classification_binary: 2.2403 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0862 | Regression_eta: 0.0034 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5345 | Classification_binary: 2.2403 | Regression_cos_phi: 1.4572 | Regression_energy: 1.7611 | Regression_eta: 1.1201 | Regression_pt: 1.7568 | Regression_sin_phi: 2.4066 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.9474s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0026 | max_backward: 0.0282 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3953 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3568.9 | uss_MB: 2370.9 | child_rss_MB: 23107.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:06:36,902] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 05:06:36,951] INFO : Running validation on rank0 +rank0 [2026-11-15 05:06:36,951] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:06:36,951] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:06:37,054] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 05:06:37,133] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:06:37,212] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:06:37,261] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:06:37,306] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:06:37,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:06:37,399] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:06:37,445] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:06:37,517] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:06:37,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:06:37,675] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:06:37,722] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:06:37,768] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:06:37,813] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:06:37,861] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:06:37,908] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:06:37,976] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:06:37,980] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:06:37,981] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:06:37,983] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:06:37,985] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:06:37,986] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:06:37,988] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:06:37,989] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:06:37,991] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:06:37,992] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:06:37,994] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:06:37,996] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:06:37,997] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:06:37,999] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:06:38,000] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:06:38,002] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:06:38,950] INFO : VALIDATION | Step=32000/500000 | Train Loss=12.2766 | Valid Loss=15.9270 | Train Loss Unweighted=2.4380 | Valid Loss Unweighted=2.3375 | Stale=29 | ETA=656.5m +rank0 [2026-11-15 05:06:38,950] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:06:39,269] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:06:39,274] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:39,326] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:06:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:06:39,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:06:40,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:06:40,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:06:40,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:06:40,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:06:40,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:06:40,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:06:41,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:06:41,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:06:41,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:06:41,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:06:41,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:06:41,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:06:42,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:06:42,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:06:42,407] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:06:42,414] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:42,819] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:43,076] INFO : Plotted number of elements +rank0 [2026-11-15 05:06:44,037] INFO : Plotted jets +rank0 [2026-11-15 05:06:45,240] INFO : Plotted jet ratio +rank0 [2026-11-15 05:06:48,206] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 05:06:48,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:07:30,456] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 05:07:30,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:08:01,599] INFO : Step 33000/500000 rank0 | Train Loss: 7.2653 | Train Loss Unweighted: 1.8358 | Train Loss Components: Classification: 0.0087 | Classification_binary: 1.7041 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0587 | Regression_eta: 0.0021 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.1341 | Classification_binary: 1.7041 | Regression_cos_phi: 0.6959 | Regression_energy: 1.1990 | Regression_eta: 0.7036 | Regression_pt: 1.1952 | Regression_sin_phi: 0.6334 | LR: 7.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8713s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.3202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3593.3 | uss_MB: 2395.3 | child_rss_MB: 23107.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:08:01,600] INFO : Running validation on rank0 +rank0 [2026-11-15 05:08:01,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:08:01,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:08:01,704] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 05:08:01,782] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:08:01,861] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:08:01,911] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:08:01,957] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:08:02,003] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:08:02,050] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:08:02,097] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:08:02,168] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:08:02,246] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:08:02,325] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:08:02,371] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:08:02,418] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:08:02,464] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:08:02,510] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:08:02,556] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:08:02,624] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:08:02,628] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:08:02,630] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:08:02,633] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:08:02,634] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:08:02,636] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:08:02,637] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:08:02,639] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:08:02,641] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:08:02,642] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:08:02,644] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:08:02,646] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:08:02,647] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:08:02,653] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:08:02,655] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:08:02,657] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:08:03,610] INFO : VALIDATION | Step=33000/500000 | Train Loss=7.2653 | Valid Loss=16.1339 | Train Loss Unweighted=1.8358 | Valid Loss Unweighted=2.3402 | Stale=30 | ETA=655.2m +rank0 [2026-11-15 05:08:03,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:08:03,787] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:08:03,796] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:03,847] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:08:04,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:08:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:08:04,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:08:04,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:08:04,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:08:05,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:08:05,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:08:05,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:08:05,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:08:05,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:08:05,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:08:06,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:08:06,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:08:06,484] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:08:06,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:08:06,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:08:06,892] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:08:06,900] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:07,296] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:07,552] INFO : Plotted number of elements +rank0 [2026-11-15 05:08:08,482] INFO : Plotted jets +rank0 [2026-11-15 05:08:09,689] INFO : Plotted jet ratio +rank0 [2026-11-15 05:08:10,083] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 05:08:10,083] INFO : Training completed. Total time on device 0: 46.408min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..d0d1793c326c95ce2a43514873b088a0a45acc6d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/train.log.0 @@ -0,0 +1,2628 @@ +rank0 [2026-11-15 04:21:44,261] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 04:21:44,261] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 04:21:44,261] INFO : Instantiating model +rank0 [2026-11-15 04:21:44,279] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 04:21:44,280] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 04:21:44,281] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 04:21:44,281] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 04:21:44,281] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 04:21:44,281] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 04:21:44,281] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,283] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 04:21:44,283] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,285] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 04:21:44,285] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,287] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 04:21:44,288] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,289] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 04:21:44,289] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,292] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 04:21:44,292] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,294] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 04:21:44,294] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 04:21:44,294] INFO : Initializing output DNNs +rank0 [2026-11-15 04:21:44,295] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,297] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:21:44,303] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 04:21:44,304] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 04:21:44,304] INFO : jagged_attention enabled=True +rank0 [2026-11-15 04:21:44,304] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 04:21:44,304] INFO : task_queries enabled=True +rank0 [2026-11-15 04:21:44,304] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 04:21:44,304] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 04:21:44,304] INFO : classification_query parameters: 256 +rank0 [2026-11-15 04:21:44,304] INFO : regression_query parameters: 256 +rank0 [2026-11-15 04:21:44,304] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 04:21:44,304] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 04:21:44,304] INFO : output_mode=elementwise +rank0 [2026-11-15 04:21:44,304] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 04:21:44,304] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 04:21:44,304] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 04:21:44,304] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 04:21:44,305] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 04:21:44,305] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 04:21:44,305] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 04:21:44,305] INFO : MLPF __init__ done +rank0 [2026-11-15 04:21:44,306] INFO : Instantiated model +rank0 [2026-11-15 04:21:44,306] INFO : Moving model to device rank=0 +rank0 [2026-11-15 04:21:44,312] INFO : Moved model to device rank=0 +rank0 [2026-11-15 04:21:44,313] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 04:21:44,388] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 04:21:44,388] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 04:21:44,388] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 04:21:44,448] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 04:21:44,448] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 04:21:44,449] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 04:21:44,449] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 04:21:44,449] INFO : Total parameters: 3910930 +rank0 [2026-11-15 04:21:44,451] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 04:21:44,452] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951 +rank0 [2026-11-15 04:21:44,452] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951 +rank0 [2026-11-15 04:21:44,452] INFO : Getting dataloaders +rank0 [2026-11-15 04:21:44,452] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:44,630] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,696] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,742] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,798] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,844] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:44,877] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,064] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,112] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,152] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,191] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:21:45,191] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 04:21:45,191] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 04:21:45,192] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 04:21:45,192] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:21:45,202] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,212] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,223] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,235] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,247] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,258] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,267] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,277] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,286] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,298] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:21:45,298] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 04:21:45,298] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 04:21:45,339] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:45,592] INFO : Got dataloaders +rank0 [2026-11-15 04:21:45,592] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 04:21:45,592] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 04:21:45,593] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 04:21:45,597] INFO : Creating train_iterator +rank0 [2026-11-15 04:21:45,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:21:45,634] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:21:45,806] INFO : Created train_iterator +rank0 [2026-11-15 04:21:54,553] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9476827490329742, 'Classification': 0.014934704909101128, 'Regression_pt': 0.0958048278465867, 'Regression_eta': 0.005884120861301198, 'Regression_sin_phi': 0.005008649970404804, 'Regression_cos_phi': 0.004921804838813841, 'Regression_energy': 0.09537884149700403}, weights={'Classification_binary': 1.0, 'Classification': 130.4132080078125, 'Regression_pt': 20.329692840576172, 'Regression_eta': 331.006591796875, 'Regression_sin_phi': 388.86383056640625, 'Regression_cos_phi': 395.7253112792969, 'Regression_energy': 20.420490264892578} +rank0 [2026-11-15 04:22:29,176] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 04:22:29,176] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:02,603] INFO : Step 1000/500000 rank0 | Train Loss: 14.8141 | Train Loss Unweighted: 2.2866 | Train Loss Components: Classification: 0.0148 | Classification_binary: 2.0414 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1069 | Regression_eta: 0.0037 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9354 | Classification_binary: 2.0414 | Regression_cos_phi: 2.0280 | Regression_energy: 2.1837 | Regression_eta: 1.2158 | Regression_pt: 2.1620 | Regression_sin_phi: 3.2478 | LR: 2.01e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7849s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0824 | max_tensorboard_logging: 0.2585 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2628.5 | uss_MB: 1357.2 | child_rss_MB: 23089.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:23:02,604] INFO : Running validation on rank0 +rank0 [2026-11-15 04:23:02,604] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:23:02,604] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:23:02,727] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 04:23:02,842] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:23:02,898] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:23:02,937] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:23:02,977] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:23:03,010] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:23:03,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:23:03,088] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:23:03,123] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:23:03,178] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:23:03,232] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:23:03,270] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:23:03,309] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:23:03,342] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:23:03,381] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:23:03,420] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:23:03,454] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:23:03,456] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:23:03,458] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:23:03,459] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:23:03,461] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:23:03,462] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:23:03,464] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:23:03,466] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:23:03,467] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:23:03,469] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:23:03,470] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:23:03,472] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:23:03,474] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:23:03,475] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:23:03,477] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:23:03,478] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:23:04,467] INFO : VALIDATION | Step=1000/500000 | Train Loss=14.8141 | Valid Loss=13.9802 | Train Loss Unweighted=2.2866 | Valid Loss Unweighted=2.1865 | Stale=0 | ETA=655.9m +rank0 [2026-11-15 04:23:04,467] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:04,597] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:23:04,605] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:04,651] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:05,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:23:05,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:23:05,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:23:05,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:23:05,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:23:05,976] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:23:06,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:23:06,301] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:23:06,462] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:23:06,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:23:06,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:23:06,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:23:07,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:23:07,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:23:07,453] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:23:07,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:23:07,677] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:23:07,685] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:08,077] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:23:08,392] INFO : Plotted number of elements +rank0 [2026-11-15 04:23:09,347] INFO : Plotted jets +rank0 [2026-11-15 04:23:10,484] INFO : Plotted jet ratio +rank0 [2026-11-15 04:23:20,184] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 04:23:20,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:02,130] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 04:24:02,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:24:25,711] INFO : Step 2000/500000 rank0 | Train Loss: 14.5811 | Train Loss Unweighted: 2.4369 | Train Loss Components: Classification: 0.0159 | Classification_binary: 2.1935 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1060 | Regression_eta: 0.0058 | Regression_pt: 0.1054 | Regression_sin_phi: 0.0044 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0741 | Classification_binary: 2.1935 | Regression_cos_phi: 2.3744 | Regression_energy: 2.1649 | Regression_eta: 1.9223 | Regression_pt: 2.1419 | Regression_sin_phi: 1.7100 | LR: 2.02e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.9152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0255 | max_loss: 0.0025 | max_backward: 0.0294 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2784.9 | uss_MB: 1529.1 | child_rss_MB: 23095.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:24:25,713] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 04:24:25,765] INFO : Running validation on rank0 +rank0 [2026-11-15 04:24:25,765] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:24:25,765] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:24:25,869] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 04:24:25,949] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:24:26,027] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:24:26,075] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:24:26,121] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:24:26,166] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:24:26,212] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:24:26,258] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:24:26,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:24:26,406] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:24:26,483] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:24:26,531] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:24:26,576] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:24:26,621] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:24:26,667] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:24:26,713] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:24:26,778] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:24:26,783] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:24:26,784] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:24:26,786] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:24:26,788] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:24:26,791] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:24:26,791] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:24:26,793] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:24:26,796] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:24:26,797] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:24:26,798] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:24:26,800] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:24:26,803] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:24:26,803] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:24:26,805] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:24:26,807] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:24:27,809] INFO : VALIDATION | Step=2000/500000 | Train Loss=14.5811 | Valid Loss=13.9228 | Train Loss Unweighted=2.4369 | Valid Loss Unweighted=2.1837 | Stale=0 | ETA=673.2m +rank0 [2026-11-15 04:24:27,810] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:24:28,233] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:24:28,238] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:28,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:24:28,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:24:28,907] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:24:29,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:24:29,233] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:24:29,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:24:29,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:24:29,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:24:29,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:24:30,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:24:30,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:24:30,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:24:30,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:24:30,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:24:30,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:24:31,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:24:31,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:24:31,277] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:24:31,284] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:31,665] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:24:31,918] INFO : Plotted number of elements +rank0 [2026-11-15 04:24:33,275] INFO : Plotted jets +rank0 [2026-11-15 04:24:33,991] INFO : Plotted jet ratio +rank0 [2026-11-15 04:24:52,923] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 04:24:52,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:35,089] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 04:25:35,089] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:50,039] INFO : Step 3000/500000 rank0 | Train Loss: 10.8090 | Train Loss Unweighted: 2.1529 | Train Loss Components: Classification: 0.0144 | Classification_binary: 1.9620 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0827 | Regression_eta: 0.0026 | Regression_pt: 0.0843 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.8792 | Classification_binary: 1.9620 | Regression_cos_phi: 1.5883 | Regression_energy: 1.6897 | Regression_eta: 0.8446 | Regression_pt: 1.7138 | Regression_sin_phi: 1.1315 | LR: 2.05e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8515s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0027 | max_backward: 0.0271 | max_optimizer_step: 0.0849 | max_tensorboard_logging: 0.3000 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.5 | uss_MB: 1576.6 | child_rss_MB: 23097.1 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:25:50,040] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:50,040] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:50,040] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:50,144] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 04:25:50,227] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:50,304] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:50,355] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:50,402] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:50,448] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:50,496] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:50,543] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:50,616] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:50,699] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:50,778] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:50,826] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:50,872] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:50,919] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:50,968] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:51,015] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:51,082] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:51,086] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:51,088] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:51,089] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:51,091] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:51,092] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:51,094] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:51,095] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:51,097] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:51,098] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:51,100] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:51,101] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:51,103] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:51,105] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:51,106] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:51,108] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:52,172] INFO : VALIDATION | Step=3000/500000 | Train Loss=10.8090 | Valid Loss=13.9179 | Train Loss Unweighted=2.1529 | Valid Loss Unweighted=2.1914 | Stale=0 | ETA=680.8m +rank0 [2026-11-15 04:25:52,173] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:52,318] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:52,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:52,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:52,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:52,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:53,159] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:53,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:53,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:53,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:53,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:53,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:54,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:54,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:54,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:54,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:54,803] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:54,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:55,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:55,277] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:55,361] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:55,368] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:55,757] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:56,016] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:57,387] INFO : Plotted jets +rank0 [2026-11-15 04:25:58,100] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:26,395] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 04:26:26,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:08,623] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:27:08,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:14,299] INFO : Step 4000/500000 rank0 | Train Loss: 12.2415 | Train Loss Unweighted: 1.8120 | Train Loss Components: Classification: 0.0149 | Classification_binary: 1.6091 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0869 | Regression_eta: 0.0034 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9390 | Classification_binary: 1.6091 | Regression_cos_phi: 2.0009 | Regression_energy: 1.7736 | Regression_eta: 1.1361 | Regression_pt: 1.7810 | Regression_sin_phi: 2.0019 | LR: 2.08e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7956s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0839 | max_tensorboard_logging: 0.2580 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2871.3 | uss_MB: 1608.5 | child_rss_MB: 23100.1 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:27:14,301] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:27:14,349] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:14,349] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:14,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:14,453] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:27:14,533] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:14,612] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:14,659] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:14,705] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:14,752] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:14,798] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:14,846] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:14,916] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:15,002] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:15,079] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:15,126] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:15,173] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:15,222] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:15,270] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:15,318] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:15,383] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:15,387] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:15,389] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:15,391] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:15,392] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:15,394] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:15,395] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:15,397] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:15,398] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:15,400] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:15,402] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:15,404] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:15,405] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:15,407] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:15,408] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:15,411] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:16,400] INFO : VALIDATION | Step=4000/500000 | Train Loss=12.2415 | Valid Loss=13.9565 | Train Loss Unweighted=1.8120 | Valid Loss Unweighted=2.1904 | Stale=1 | ETA=683.7m +rank0 [2026-11-15 04:27:16,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:16,540] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:16,558] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:16,607] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:17,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:17,211] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:17,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:17,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:17,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:17,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:18,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:18,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:18,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:18,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:18,712] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:18,875] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:19,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:19,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:19,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:19,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:19,601] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:27:19,609] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:19,999] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:20,260] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:21,170] INFO : Plotted jets +rank0 [2026-11-15 04:27:22,357] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:00,153] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:28:00,153] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:28:38,716] INFO : Step 5000/500000 rank0 | Train Loss: 14.3780 | Train Loss Unweighted: 2.2767 | Train Loss Components: Classification: 0.0121 | Classification_binary: 2.0676 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0880 | Regression_eta: 0.0084 | Regression_pt: 0.0896 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5726 | Classification_binary: 2.0676 | Regression_cos_phi: 2.1802 | Regression_energy: 1.7977 | Regression_eta: 2.7861 | Regression_pt: 1.8206 | Regression_sin_phi: 2.1532 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8288s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0025 | max_backward: 0.0276 | max_optimizer_step: 0.0848 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2893.4 | uss_MB: 1630.8 | child_rss_MB: 23096.4 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:28:38,717] INFO : Running validation on rank0 +rank0 [2026-11-15 04:28:38,717] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:28:38,717] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:28:38,824] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:28:38,901] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:28:38,979] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:28:39,026] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:28:39,071] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:28:39,116] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:28:39,162] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:28:39,208] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:28:39,277] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:28:39,354] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:28:39,431] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:28:39,478] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:28:39,523] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:28:39,568] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:28:39,615] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,662] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:28:39,727] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:28:39,731] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:28:39,733] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:28:39,735] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:28:39,736] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:28:39,738] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:28:39,739] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:28:39,741] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:28:39,742] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:28:39,744] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:28:39,746] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:28:39,747] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:28:39,749] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:28:39,750] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:28:39,752] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:28:39,754] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:28:40,702] INFO : VALIDATION | Step=5000/500000 | Train Loss=14.3780 | Valid Loss=13.9897 | Train Loss Unweighted=2.2767 | Valid Loss Unweighted=2.1945 | Stale=2 | ETA=684.9m +rank0 [2026-11-15 04:28:40,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:28:41,249] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:28:41,254] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:41,317] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:28:41,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:28:41,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:28:42,155] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:28:42,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:28:42,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:28:42,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:28:42,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:28:42,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:28:43,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:28:43,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:28:43,476] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:28:43,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:28:43,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:28:43,971] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:28:44,135] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:28:44,282] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:28:44,353] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:28:44,361] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:44,749] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:28:45,002] INFO : Plotted number of elements +rank0 [2026-11-15 04:28:45,928] INFO : Plotted jets +rank0 [2026-11-15 04:28:47,095] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:51,713] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:28:51,713] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:33,272] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:29:33,272] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:02,488] INFO : Step 6000/500000 rank0 | Train Loss: 13.1978 | Train Loss Unweighted: 2.3508 | Train Loss Components: Classification: 0.0161 | Classification_binary: 2.1205 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1016 | Regression_eta: 0.0056 | Regression_pt: 0.0993 | Regression_sin_phi: 0.0031 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0972 | Classification_binary: 2.1205 | Regression_cos_phi: 1.8161 | Regression_energy: 2.0740 | Regression_eta: 1.8679 | Regression_pt: 2.0179 | Regression_sin_phi: 1.2041 | LR: 2.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8132s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2583 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2839.3 | uss_MB: 1576.6 | child_rss_MB: 23100.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:02,490] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:30:02,540] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:02,541] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:02,541] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:02,659] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:30:02,740] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:02,819] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:02,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:02,913] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:02,958] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:03,005] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,051] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:03,122] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:03,201] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:03,279] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:03,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:03,372] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:03,417] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:03,462] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:03,508] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:03,573] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:03,577] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:03,579] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:03,580] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:03,582] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:03,583] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:03,585] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:03,587] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:03,588] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:03,590] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:03,591] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:03,593] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:03,594] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:03,596] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:03,597] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:03,599] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:04,566] INFO : VALIDATION | Step=6000/500000 | Train Loss=13.1978 | Valid Loss=14.0344 | Train Loss Unweighted=2.3508 | Valid Loss Unweighted=2.1991 | Stale=3 | ETA=684.7m +rank0 [2026-11-15 04:30:04,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:04,702] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:04,709] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:04,771] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:05,207] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:05,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:05,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:05,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:05,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:06,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:06,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:06,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:06,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:06,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:06,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:07,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:07,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:07,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:07,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:07,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:07,775] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:07,783] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:08,168] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:08,424] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:09,798] INFO : Plotted jets +rank0 [2026-11-15 04:30:10,516] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:24,517] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:30:24,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:06,476] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:31:06,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:26,389] INFO : Step 7000/500000 rank0 | Train Loss: 11.9085 | Train Loss Unweighted: 2.0755 | Train Loss Components: Classification: 0.0152 | Classification_binary: 1.8821 | Regression_cos_phi: 0.0040 | Regression_energy: 0.0816 | Regression_eta: 0.0033 | Regression_pt: 0.0842 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9879 | Classification_binary: 1.8821 | Regression_cos_phi: 1.5839 | Regression_energy: 1.6662 | Regression_eta: 1.0917 | Regression_pt: 1.7120 | Regression_sin_phi: 1.9848 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8269s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0027 | max_backward: 0.0324 | max_optimizer_step: 0.0888 | max_tensorboard_logging: 0.2522 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2948.6 | uss_MB: 1685.9 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:31:26,391] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:26,391] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:26,391] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:26,500] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:31:26,580] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:26,659] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:26,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:26,759] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:26,805] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:26,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:26,899] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:26,970] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:27,051] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:27,137] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:27,184] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:27,230] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:27,277] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:27,325] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:27,372] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:27,440] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:27,444] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:27,446] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:27,448] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:27,449] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:27,451] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:27,452] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:27,454] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:27,455] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:27,458] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:27,458] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:27,460] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:27,461] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:27,463] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:27,464] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:27,467] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:28,443] INFO : VALIDATION | Step=7000/500000 | Train Loss=11.9085 | Valid Loss=14.0624 | Train Loss Unweighted=2.0755 | Valid Loss Unweighted=2.1997 | Stale=4 | ETA=684.2m +rank0 [2026-11-15 04:31:28,444] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:29,008] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:29,013] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:29,068] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:29,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:29,755] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:29,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:30,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:30,245] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:30,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:30,571] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:30,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:30,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:31,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:31,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:31,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:31,578] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:31,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:31,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:32,054] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:32,148] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:32,159] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:32,550] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:32,806] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:34,208] INFO : Plotted jets +rank0 [2026-11-15 04:31:34,933] INFO : Plotted jet ratio +rank0 [2026-11-15 04:31:58,670] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:31:58,671] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:40,629] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:32:40,629] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:51,262] INFO : Step 8000/500000 rank0 | Train Loss: 11.3475 | Train Loss Unweighted: 2.5754 | Train Loss Components: Classification: 0.0131 | Classification_binary: 2.3900 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0815 | Regression_eta: 0.0028 | Regression_pt: 0.0803 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7109 | Classification_binary: 2.3900 | Regression_cos_phi: 1.5285 | Regression_energy: 1.6634 | Regression_eta: 0.9242 | Regression_pt: 1.6328 | Regression_sin_phi: 1.4977 | LR: 2.34e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0274 | max_optimizer_step: 0.0840 | max_tensorboard_logging: 0.3040 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2968.7 | uss_MB: 1706.1 | child_rss_MB: 23102.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:32:51,264] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:32:51,315] INFO : Running validation on rank0 +rank0 [2026-11-15 04:32:51,315] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:32:51,315] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:32:51,420] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:32:51,501] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:32:51,580] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:32:51,629] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:32:51,674] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:32:51,719] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:32:51,766] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:32:51,812] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:32:51,882] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:32:51,960] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:32:52,037] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:32:52,084] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:32:52,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:32:52,176] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:32:52,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:32:52,267] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:32:52,333] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:32:52,338] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:32:52,339] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:32:52,341] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:32:52,342] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:32:52,344] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:32:52,346] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:32:52,347] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:32:52,349] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:32:52,351] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:32:52,353] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:32:52,354] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:32:52,356] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:32:52,357] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:32:52,359] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:32:52,362] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:32:53,338] INFO : VALIDATION | Step=8000/500000 | Train Loss=11.3475 | Valid Loss=14.1343 | Train Loss Unweighted=2.5754 | Valid Loss Unweighted=2.2088 | Stale=5 | ETA=684.4m +rank0 [2026-11-15 04:32:53,338] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:32:53,991] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:32:53,999] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:54,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:32:54,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:32:54,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:32:54,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:32:55,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:32:55,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:32:55,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:32:55,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:32:55,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:32:55,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:32:56,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:32:56,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:32:56,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:32:56,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:32:56,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:32:56,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:32:56,999] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:32:57,071] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:32:57,078] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:57,463] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:32:57,716] INFO : Plotted number of elements +rank0 [2026-11-15 04:32:59,081] INFO : Plotted jets +rank0 [2026-11-15 04:32:59,817] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:31,525] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:33:31,526] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:13,084] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:34:13,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:14,440] INFO : Step 9000/500000 rank0 | Train Loss: 13.1962 | Train Loss Unweighted: 2.1812 | Train Loss Components: Classification: 0.0156 | Classification_binary: 1.9737 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0883 | Regression_eta: 0.0047 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0389 | Classification_binary: 1.9737 | Regression_cos_phi: 2.2095 | Regression_energy: 1.8040 | Regression_eta: 1.5592 | Regression_pt: 1.8001 | Regression_sin_phi: 1.8107 | LR: 2.43e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8269s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2912.8 | uss_MB: 1650.3 | child_rss_MB: 23101.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:34:14,442] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:14,442] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:14,442] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:14,561] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:34:14,644] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:14,723] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:14,774] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:14,821] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:14,867] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:14,913] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:14,960] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:15,031] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:15,110] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:15,190] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:15,238] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:15,284] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:15,329] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:15,376] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:15,421] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:15,488] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:15,493] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:15,494] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:15,496] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:15,498] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:15,499] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:15,501] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:15,502] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:15,504] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:15,505] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:15,507] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:15,509] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:15,510] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:15,512] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:15,513] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:15,515] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:16,488] INFO : VALIDATION | Step=9000/500000 | Train Loss=13.1962 | Valid Loss=14.2057 | Train Loss Unweighted=2.1812 | Valid Loss Unweighted=2.2139 | Stale=6 | ETA=682.8m +rank0 [2026-11-15 04:34:16,489] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:16,626] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:16,635] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:16,691] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:17,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:17,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:17,460] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:17,620] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:17,785] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:17,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:18,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:18,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:18,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:18,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:18,783] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:18,951] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:19,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:19,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:19,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:19,580] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:19,665] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:19,673] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:20,063] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:20,328] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:21,262] INFO : Plotted jets +rank0 [2026-11-15 04:34:22,430] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:04,118] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:35:04,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:38,396] INFO : Step 10000/500000 rank0 | Train Loss: 16.4031 | Train Loss Unweighted: 2.4290 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.1766 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1068 | Regression_eta: 0.0096 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.2988 | Classification_binary: 2.1766 | Regression_cos_phi: 2.5845 | Regression_energy: 2.1800 | Regression_eta: 3.1889 | Regression_pt: 2.1796 | Regression_sin_phi: 1.7946 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8442s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0026 | max_backward: 0.0282 | max_optimizer_step: 0.0814 | max_tensorboard_logging: 0.2986 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3022.0 | uss_MB: 1759.4 | child_rss_MB: 23102.9 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:38,397] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:35:38,444] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:38,444] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:38,444] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:38,547] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:35:38,628] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:38,706] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:38,755] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:38,800] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:38,845] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:38,891] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:38,937] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:39,007] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:39,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:39,162] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:39,209] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:39,254] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:39,300] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:39,347] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:39,393] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:39,458] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:39,463] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:39,464] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:39,466] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:39,467] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:39,469] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:39,471] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:39,472] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:39,474] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:39,475] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:39,477] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:39,478] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:39,480] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:39,481] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:39,483] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:39,484] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:35:40,434] INFO : VALIDATION | Step=10000/500000 | Train Loss=16.4031 | Valid Loss=14.2347 | Train Loss Unweighted=2.4290 | Valid Loss Unweighted=2.2174 | Stale=7 | ETA=681.8m +rank0 [2026-11-15 04:35:40,435] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:35:40,823] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:35:40,832] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:40,881] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:35:41,350] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:35:41,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:35:41,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:35:41,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:35:42,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:35:42,198] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:35:42,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:35:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:35:42,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:35:42,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:35:43,030] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:35:43,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:35:43,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:35:43,556] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:35:43,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:35:43,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:35:43,970] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:35:43,977] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:44,369] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:35:44,638] INFO : Plotted number of elements +rank0 [2026-11-15 04:35:46,067] INFO : Plotted jets +rank0 [2026-11-15 04:35:46,802] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:55,923] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:35:55,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:36:38,450] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:36:38,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:03,542] INFO : Step 11000/500000 rank0 | Train Loss: 15.1416 | Train Loss Unweighted: 2.1507 | Train Loss Components: Classification: 0.0187 | Classification_binary: 1.9016 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1041 | Regression_eta: 0.0070 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.4422 | Classification_binary: 1.9016 | Regression_cos_phi: 1.9325 | Regression_energy: 2.1260 | Regression_eta: 2.3024 | Regression_pt: 2.2093 | Regression_sin_phi: 2.2276 | LR: 2.63e-05 | DataLoad Time: 0.0112s | Model Forward Time: 0.8326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0900 | max_tensorboard_logging: 0.2739 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3052.0 | uss_MB: 1789.4 | child_rss_MB: 23105.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 25.6 +rank0 [2026-11-15 04:37:03,543] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:03,543] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:03,543] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:03,648] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:37:03,727] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:03,806] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:03,854] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:03,899] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:03,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:03,991] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:04,038] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:04,108] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:04,185] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:04,263] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:04,311] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:04,357] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:04,403] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:04,450] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:04,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:04,563] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:04,568] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:04,569] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:04,571] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:04,573] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:04,574] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:04,576] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:04,577] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:04,579] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:04,580] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:04,582] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:04,583] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:04,585] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:04,586] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:04,588] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:04,592] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:05,540] INFO : VALIDATION | Step=11000/500000 | Train Loss=15.1416 | Valid Loss=14.2935 | Train Loss Unweighted=2.1507 | Valid Loss Unweighted=2.2165 | Stale=8 | ETA=681.6m +rank0 [2026-11-15 04:37:05,540] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:05,679] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:05,687] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:05,736] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:06,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:06,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:06,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:06,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:06,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:07,026] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:07,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:07,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:07,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:07,678] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:07,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:08,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:08,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:08,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:08,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:08,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:08,731] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:08,738] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:09,131] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:09,385] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:10,306] INFO : Plotted jets +rank0 [2026-11-15 04:37:11,478] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:29,813] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:37:29,814] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:12,143] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:38:12,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:27,747] INFO : Step 12000/500000 rank0 | Train Loss: 14.0491 | Train Loss Unweighted: 2.1502 | Train Loss Components: Classification: 0.0160 | Classification_binary: 1.9716 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0714 | Regression_eta: 0.0070 | Regression_pt: 0.0721 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0868 | Classification_binary: 1.9716 | Regression_cos_phi: 2.7559 | Regression_energy: 1.4581 | Regression_eta: 2.3095 | Regression_pt: 1.4660 | Regression_sin_phi: 2.0013 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0279 | max_loss: 0.0025 | max_backward: 0.0277 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2863 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3061.5 | uss_MB: 1798.8 | child_rss_MB: 23104.1 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:38:27,749] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:38:27,797] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:27,797] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:27,797] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:27,902] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:38:27,983] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:28,061] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:28,114] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:28,162] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:28,207] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:28,255] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:28,304] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:28,375] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:28,453] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:28,531] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:28,577] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:28,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:28,668] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:28,715] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:28,761] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:28,825] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:28,830] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:28,831] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:28,833] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:28,834] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:28,836] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:28,837] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:28,839] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:28,841] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:28,842] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:28,844] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:28,845] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:28,847] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:28,848] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:28,850] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:28,851] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:29,837] INFO : VALIDATION | Step=12000/500000 | Train Loss=14.0491 | Valid Loss=14.2685 | Train Loss Unweighted=2.1502 | Valid Loss Unweighted=2.2237 | Stale=9 | ETA=680.7m +rank0 [2026-11-15 04:38:29,838] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:30,298] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:30,306] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:30,356] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:30,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:31,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:31,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:31,342] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:31,506] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:31,670] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:31,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:31,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:32,161] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:32,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:32,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:32,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:32,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:32,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:33,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:33,300] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:33,411] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:33,418] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:33,809] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:34,062] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:34,981] INFO : Plotted jets +rank0 [2026-11-15 04:38:36,162] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:03,891] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:39:03,891] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:46,496] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:39:46,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:52,865] INFO : Step 13000/500000 rank0 | Train Loss: 13.9720 | Train Loss Unweighted: 2.4257 | Train Loss Components: Classification: 0.0157 | Classification_binary: 2.1901 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1019 | Regression_eta: 0.0068 | Regression_pt: 0.1029 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.0505 | Classification_binary: 2.1901 | Regression_cos_phi: 2.1612 | Regression_energy: 2.0804 | Regression_eta: 2.2352 | Regression_pt: 2.0910 | Regression_sin_phi: 1.1635 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8698s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0026 | max_backward: 0.0278 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.3186 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3099.4 | uss_MB: 1836.8 | child_rss_MB: 23102.6 | allocated_MB: 170.9 | reserved_MB: 672.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:39:52,867] INFO : Running validation on rank0 +rank0 [2026-11-15 04:39:52,867] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:39:52,867] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:39:52,975] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:39:53,055] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:39:53,133] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:39:53,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:39:53,228] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:39:53,274] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:39:53,320] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:39:53,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:39:53,437] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:39:53,514] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:39:53,591] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:39:53,638] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:39:53,684] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:39:53,729] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:39:53,775] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:39:53,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:39:53,886] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:39:53,890] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:39:53,892] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:39:53,894] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:39:53,895] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:39:53,897] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:39:53,899] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:39:53,900] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:39:53,902] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:39:53,903] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:39:53,905] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:39:53,906] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:39:53,908] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:39:53,910] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:39:53,911] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:39:53,913] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:39:54,882] INFO : VALIDATION | Step=13000/500000 | Train Loss=13.9720 | Valid Loss=14.3628 | Train Loss Unweighted=2.4257 | Valid Loss Unweighted=2.2238 | Stale=10 | ETA=680.1m +rank0 [2026-11-15 04:39:54,883] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:39:55,006] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:39:55,015] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:55,072] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:39:55,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:39:55,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:39:55,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:39:56,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:39:56,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:39:56,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:39:56,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:39:56,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:39:56,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:39:57,031] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:39:57,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:39:57,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:39:57,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:39:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:39:57,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:39:58,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:39:58,160] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:39:58,167] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:58,559] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:39:58,813] INFO : Plotted number of elements +rank0 [2026-11-15 04:39:59,732] INFO : Plotted jets +rank0 [2026-11-15 04:40:00,919] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:37,878] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:40:37,878] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:16,960] INFO : Step 14000/500000 rank0 | Train Loss: 10.5674 | Train Loss Unweighted: 2.2669 | Train Loss Components: Classification: 0.0113 | Classification_binary: 2.0889 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0753 | Regression_eta: 0.0039 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0023 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.4753 | Classification_binary: 2.0889 | Regression_cos_phi: 1.6410 | Regression_energy: 1.5383 | Regression_eta: 1.2984 | Regression_pt: 1.6481 | Regression_sin_phi: 0.8774 | LR: 3.02e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.9189s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0025 | max_backward: 0.0279 | max_optimizer_step: 0.0923 | max_tensorboard_logging: 0.3469 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3125.6 | uss_MB: 1862.9 | child_rss_MB: 23104.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:16,962] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:41:17,012] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:17,012] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:17,012] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:17,119] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:41:17,201] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:17,280] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:17,329] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:17,378] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:17,428] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:17,475] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:17,522] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:17,593] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:17,673] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:17,751] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:17,799] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:17,845] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:17,890] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:17,937] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:17,983] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:18,050] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:18,055] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:18,056] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:18,058] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:18,059] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:18,061] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:18,063] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:18,064] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:18,066] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:18,067] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:18,069] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:18,070] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:18,073] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:18,073] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:18,075] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:18,076] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:19,064] INFO : VALIDATION | Step=14000/500000 | Train Loss=10.5674 | Valid Loss=14.4554 | Train Loss Unweighted=2.2669 | Valid Loss Unweighted=2.2346 | Stale=11 | ETA=678.9m +rank0 [2026-11-15 04:41:19,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:19,523] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:19,534] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:19,592] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:20,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:20,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:20,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:20,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:20,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:20,962] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:21,130] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:21,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:21,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:21,624] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:21,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:21,950] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:22,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:22,275] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:22,436] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:22,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:22,668] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:22,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:23,058] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:23,322] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:24,238] INFO : Plotted jets +rank0 [2026-11-15 04:41:25,435] INFO : Plotted jet ratio +rank0 [2026-11-15 04:41:29,595] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:41:29,595] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:11,481] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:42:11,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:40,967] INFO : Step 15000/500000 rank0 | Train Loss: 16.4500 | Train Loss Unweighted: 2.2700 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.0440 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0911 | Regression_eta: 0.0077 | Regression_pt: 0.0954 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 2.1864 | Classification_binary: 2.0440 | Regression_cos_phi: 2.6354 | Regression_energy: 1.8603 | Regression_eta: 2.5484 | Regression_pt: 1.9402 | Regression_sin_phi: 3.2353 | LR: 3.17e-05 | DataLoad Time: 0.0034s | Model Forward Time: 0.8666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0025 | max_backward: 0.0280 | max_optimizer_step: 0.0901 | max_tensorboard_logging: 0.3065 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3151.4 | uss_MB: 1888.7 | child_rss_MB: 23109.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:42:40,968] INFO : Running validation on rank0 +rank0 [2026-11-15 04:42:40,968] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:42:40,968] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:42:41,080] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:42:41,161] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:41,240] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:41,290] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:41,339] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:41,385] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:41,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:41,478] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:41,552] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:41,632] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:41,710] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:41,758] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:41,809] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:41,857] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:41,904] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:41,950] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:42,017] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:42,022] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:42,023] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:42,025] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:42,027] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:42,028] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:42,030] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:42,032] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:42,033] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:42,035] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:42,036] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:42,038] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:42,040] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:42,041] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:42,043] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:42,044] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:43,015] INFO : VALIDATION | Step=15000/500000 | Train Loss=16.4500 | Valid Loss=14.5445 | Train Loss Unweighted=2.2700 | Valid Loss Unweighted=2.2374 | Stale=12 | ETA=677.6m +rank0 [2026-11-15 04:42:43,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:43,154] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:43,181] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:43,242] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:43,726] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:43,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:44,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:44,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:44,398] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:44,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:44,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:44,897] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:45,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:45,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:45,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:45,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:45,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:45,911] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:46,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:46,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:46,362] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:42:46,369] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:46,765] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:47,023] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:47,991] INFO : Plotted jets +rank0 [2026-11-15 04:42:49,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:43:02,826] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:43:02,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:44,991] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:43:44,991] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:44:05,807] INFO : Step 16000/500000 rank0 | Train Loss: 9.9865 | Train Loss Unweighted: 2.1048 | Train Loss Components: Classification: 0.0148 | Classification_binary: 1.9383 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0715 | Regression_eta: 0.0024 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9280 | Classification_binary: 1.9383 | Regression_cos_phi: 1.0220 | Regression_energy: 1.4593 | Regression_eta: 0.7914 | Regression_pt: 1.4568 | Regression_sin_phi: 1.3908 | LR: 3.33e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8544s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0026 | max_backward: 0.0265 | max_optimizer_step: 0.0849 | max_tensorboard_logging: 0.2984 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3175.7 | uss_MB: 1913.1 | child_rss_MB: 23111.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:44:05,809] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:44:05,858] INFO : Running validation on rank0 +rank0 [2026-11-15 04:44:05,858] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:44:05,858] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:44:05,972] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:44:06,052] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:44:06,132] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:44:06,180] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:44:06,227] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:44:06,273] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:44:06,321] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:44:06,370] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:44:06,442] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:44:06,521] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:44:06,598] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:44:06,647] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:44:06,693] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:44:06,740] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:44:06,787] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:44:06,839] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:44:06,905] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:44:06,909] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:44:06,911] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:44:06,914] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:44:06,914] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:44:06,917] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:44:06,919] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:44:06,919] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:44:06,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:44:06,923] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:44:06,924] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:44:06,925] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:44:06,927] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:44:06,930] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:44:06,930] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:44:06,932] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:07,894] INFO : VALIDATION | Step=16000/500000 | Train Loss=9.9865 | Valid Loss=14.5846 | Train Loss Unweighted=2.1048 | Valid Loss Unweighted=2.2441 | Stale=13 | ETA=676.7m +rank0 [2026-11-15 04:44:07,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:08,314] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:08,323] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:08,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:08,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:09,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:09,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:09,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:09,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:09,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:09,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:10,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:10,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:10,390] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:10,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:10,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:10,894] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:11,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:11,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:11,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:11,450] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:44:11,457] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:11,831] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:12,084] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:13,028] INFO : Plotted jets +rank0 [2026-11-15 04:44:14,217] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:36,395] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:44:36,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:18,107] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:45:18,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:29,347] INFO : Step 17000/500000 rank0 | Train Loss: 13.2417 | Train Loss Unweighted: 2.1527 | Train Loss Components: Classification: 0.0107 | Classification_binary: 1.9703 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0772 | Regression_eta: 0.0063 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3977 | Classification_binary: 1.9703 | Regression_cos_phi: 1.9962 | Regression_energy: 1.5771 | Regression_eta: 2.0956 | Regression_pt: 1.5511 | Regression_sin_phi: 2.6538 | LR: 3.51e-05 | DataLoad Time: 0.0083s | Model Forward Time: 0.8214s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0025 | max_backward: 0.0278 | max_optimizer_step: 0.0901 | max_tensorboard_logging: 0.2545 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3111.9 | uss_MB: 1849.3 | child_rss_MB: 23105.8 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:45:29,349] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:29,349] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:29,349] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:29,460] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:45:29,543] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:29,623] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:29,673] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:29,719] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:29,765] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:29,811] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:29,858] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:29,933] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:30,011] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:30,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:30,138] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:30,184] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:30,230] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:30,277] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:30,327] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:30,394] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:30,399] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:30,400] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:30,402] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:30,404] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:30,405] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:30,407] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:30,408] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:30,410] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:30,411] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:30,413] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:30,414] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:30,416] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:30,417] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:30,419] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:30,421] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:31,387] INFO : VALIDATION | Step=17000/500000 | Train Loss=13.2417 | Valid Loss=14.7355 | Train Loss Unweighted=2.1527 | Valid Loss Unweighted=2.2478 | Stale=14 | ETA=675.2m +rank0 [2026-11-15 04:45:31,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:31,537] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:31,546] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:31,598] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:32,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:32,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:32,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:32,570] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:32,734] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:32,899] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:33,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:33,224] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:33,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:33,545] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:33,708] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:33,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:34,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:34,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:34,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:34,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:34,623] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:45:34,631] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:35,011] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:35,265] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:36,189] INFO : Plotted jets +rank0 [2026-11-15 04:45:37,381] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:09,755] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:46:09,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:52,101] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:46:52,101] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:54,073] INFO : Step 18000/500000 rank0 | Train Loss: 13.2638 | Train Loss Unweighted: 2.1536 | Train Loss Components: Classification: 0.0151 | Classification_binary: 1.9378 | Regression_cos_phi: 0.0054 | Regression_energy: 0.0916 | Regression_eta: 0.0037 | Regression_pt: 0.0943 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9729 | Classification_binary: 1.9378 | Regression_cos_phi: 2.1334 | Regression_energy: 1.8714 | Regression_eta: 1.2239 | Regression_pt: 1.9171 | Regression_sin_phi: 2.2074 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.8364s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0256 | max_loss: 0.0040 | max_backward: 0.0282 | max_optimizer_step: 0.0902 | max_tensorboard_logging: 0.2684 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3224.3 | uss_MB: 1961.7 | child_rss_MB: 23107.5 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 13.1 +rank0 [2026-11-15 04:46:54,074] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:46:54,123] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:54,123] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:54,123] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:54,231] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:46:54,318] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:54,398] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:54,449] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:54,495] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:54,545] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:54,593] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:54,639] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:54,711] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:54,790] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:54,867] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:54,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:54,962] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:55,009] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:55,058] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:55,107] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:55,174] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:55,179] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:55,180] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:55,182] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:55,185] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:55,185] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:55,187] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:55,188] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:55,190] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:55,191] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:55,193] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:55,194] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:55,196] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:55,197] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:55,199] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:55,201] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:46:56,177] INFO : VALIDATION | Step=18000/500000 | Train Loss=13.2638 | Valid Loss=14.6714 | Train Loss Unweighted=2.1536 | Valid Loss Unweighted=2.2536 | Stale=15 | ETA=674.2m +rank0 [2026-11-15 04:46:56,178] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:46:56,311] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:46:56,319] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:56,371] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:46:56,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:46:57,023] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:46:57,192] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:46:57,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:46:57,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:46:57,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:46:57,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:46:58,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:46:58,176] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:46:58,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:46:58,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:46:58,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:46:58,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:46:59,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:46:59,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:46:59,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:46:59,464] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:46:59,483] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:46:59,879] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:00,141] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:01,084] INFO : Plotted jets +rank0 [2026-11-15 04:47:02,301] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:43,336] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:47:43,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:18,231] INFO : Step 19000/500000 rank0 | Train Loss: 11.1812 | Train Loss Unweighted: 2.2649 | Train Loss Components: Classification: 0.0147 | Classification_binary: 2.0556 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0928 | Regression_eta: 0.0030 | Regression_pt: 0.0925 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9204 | Classification_binary: 2.0556 | Regression_cos_phi: 1.1161 | Regression_energy: 1.8944 | Regression_eta: 0.9884 | Regression_pt: 1.8813 | Regression_sin_phi: 1.3250 | LR: 3.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8149s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2542 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3250.3 | uss_MB: 1987.7 | child_rss_MB: 23104.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:48:18,232] INFO : Running validation on rank0 +rank0 [2026-11-15 04:48:18,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:48:18,232] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:48:18,339] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:48:18,420] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:48:18,499] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:48:18,550] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:48:18,597] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:48:18,643] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:48:18,690] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:48:18,736] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:48:18,808] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:48:18,886] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:48:18,964] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:48:19,013] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:48:19,059] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:48:19,106] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:48:19,152] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:48:19,198] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:48:19,265] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:48:19,269] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:48:19,271] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:48:19,273] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:48:19,274] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:48:19,276] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:48:19,277] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:48:19,279] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:48:19,280] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:48:19,282] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:48:19,284] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:48:19,285] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:48:19,287] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:48:19,288] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:48:19,290] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:48:19,292] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:48:20,266] INFO : VALIDATION | Step=19000/500000 | Train Loss=11.1812 | Valid Loss=14.8614 | Train Loss Unweighted=2.2649 | Valid Loss Unweighted=2.2595 | Stale=16 | ETA=672.8m +rank0 [2026-11-15 04:48:20,266] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:48:20,428] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:48:20,436] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:20,500] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:48:21,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:48:21,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:48:21,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:48:21,519] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:48:21,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:48:21,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:48:22,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:22,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:22,357] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:22,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:22,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:22,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:23,027] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:23,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:23,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:23,505] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:23,633] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:48:23,641] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:24,016] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:24,269] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:25,201] INFO : Plotted jets +rank0 [2026-11-15 04:48:26,381] INFO : Plotted jet ratio +rank0 [2026-11-15 04:48:34,928] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:48:34,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:17,279] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:49:17,279] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:42,857] INFO : Step 20000/500000 rank0 | Train Loss: 13.5737 | Train Loss Unweighted: 1.9784 | Train Loss Components: Classification: 0.0134 | Classification_binary: 1.7792 | Regression_cos_phi: 0.0056 | Regression_energy: 0.0806 | Regression_eta: 0.0067 | Regression_pt: 0.0873 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7436 | Classification_binary: 1.7792 | Regression_cos_phi: 2.2135 | Regression_energy: 1.6465 | Regression_eta: 2.2340 | Regression_pt: 1.7755 | Regression_sin_phi: 2.1815 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8719s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0027 | max_backward: 0.0285 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.3008 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3186.8 | uss_MB: 1924.2 | child_rss_MB: 23106.3 | allocated_MB: 170.4 | reserved_MB: 698.0 | inactive_split_MB: 27.6 +rank0 [2026-11-15 04:49:42,859] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:49:42,921] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:42,921] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:42,921] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:43,025] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:49:43,109] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:43,187] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:43,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:43,282] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:43,328] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:43,374] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:43,421] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:43,492] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:43,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:43,649] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:43,697] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:43,745] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:43,797] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:43,845] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:43,891] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:43,962] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:43,967] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:43,969] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:43,970] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:43,973] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:43,973] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:43,975] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:43,978] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:43,978] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:43,981] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:43,982] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:43,983] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:43,985] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:43,986] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:43,988] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:43,989] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:44,950] INFO : VALIDATION | Step=20000/500000 | Train Loss=13.5737 | Valid Loss=14.7951 | Train Loss Unweighted=1.9784 | Valid Loss Unweighted=2.2705 | Stale=17 | ETA=671.7m +rank0 [2026-11-15 04:49:44,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:45,364] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:45,380] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:45,448] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:45,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:46,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:46,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:46,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:46,635] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:46,797] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:46,963] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:47,126] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:47,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:47,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:47,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:47,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:49:47,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:49:48,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:49:48,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:49:48,442] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:49:48,520] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:49:48,528] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:48,918] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,172] INFO : Plotted number of elements +rank0 [2026-11-15 04:49:50,100] INFO : Plotted jets +rank0 [2026-11-15 04:49:51,300] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:09,143] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:50:09,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:50:51,099] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:50:51,099] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:07,149] INFO : Step 21000/500000 rank0 | Train Loss: 10.2156 | Train Loss Unweighted: 1.8802 | Train Loss Components: Classification: 0.0132 | Classification_binary: 1.7134 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0716 | Regression_eta: 0.0035 | Regression_pt: 0.0716 | Regression_sin_phi: 0.0026 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.7206 | Classification_binary: 1.7134 | Regression_cos_phi: 1.6729 | Regression_energy: 1.4625 | Regression_eta: 1.1601 | Regression_pt: 1.4561 | Regression_sin_phi: 1.0301 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0275 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3297.0 | uss_MB: 2034.4 | child_rss_MB: 23104.3 | allocated_MB: 170.4 | reserved_MB: 712.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:51:07,150] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:07,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:07,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:07,256] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:51:07,342] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:07,421] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:07,471] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:07,519] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:07,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:07,613] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:07,660] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:07,735] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:07,816] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:07,894] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:07,942] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:07,992] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:08,039] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:08,086] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:08,139] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:08,209] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:08,214] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:08,216] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:08,217] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:08,219] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:08,221] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:08,222] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:08,225] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:08,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:08,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:08,230] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:08,230] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:08,232] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:08,235] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:08,235] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:08,237] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:09,241] INFO : VALIDATION | Step=21000/500000 | Train Loss=10.2156 | Valid Loss=14.9642 | Train Loss Unweighted=1.8802 | Valid Loss Unweighted=2.2708 | Stale=18 | ETA=670.5m +rank0 [2026-11-15 04:51:09,242] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:09,563] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:09,568] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:09,632] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:10,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:10,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:10,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:10,658] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:10,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:10,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:11,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:11,324] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:11,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:11,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:11,830] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:11,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:12,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:12,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:12,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:12,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:12,731] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:51:12,740] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:13,116] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:13,371] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:14,321] INFO : Plotted jets +rank0 [2026-11-15 04:51:15,497] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:42,422] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:51:42,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:24,415] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:52:24,415] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:31,434] INFO : Step 22000/500000 rank0 | Train Loss: 14.4647 | Train Loss Unweighted: 2.5983 | Train Loss Components: Classification: 0.0125 | Classification_binary: 2.3248 | Regression_cos_phi: 0.0030 | Regression_energy: 0.1300 | Regression_eta: 0.0089 | Regression_pt: 0.1155 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.6298 | Classification_binary: 2.3248 | Regression_cos_phi: 1.2046 | Regression_energy: 2.6555 | Regression_eta: 2.9478 | Regression_pt: 2.3490 | Regression_sin_phi: 1.3532 | LR: 4.50e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8921s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0274 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3317.8 | uss_MB: 2055.2 | child_rss_MB: 23106.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:52:31,436] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:52:31,487] INFO : Running validation on rank0 +rank0 [2026-11-15 04:52:31,487] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:52:31,487] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:52:31,591] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:52:31,671] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:52:31,750] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:52:31,799] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:52:31,846] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:52:31,891] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:52:31,937] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:52:31,984] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:52:32,055] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:52:32,134] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:52:32,211] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:52:32,259] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:52:32,305] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:52:32,352] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:52:32,398] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:52:32,444] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:52:32,511] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:52:32,516] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:52:32,518] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:52:32,519] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:52:32,521] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:52:32,523] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:52:32,525] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:52:32,526] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:52:32,528] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:52:32,531] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:52:32,531] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:52:32,533] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:52:32,535] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:52:32,537] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:52:32,538] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:52:32,540] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:52:33,505] INFO : VALIDATION | Step=22000/500000 | Train Loss=14.4647 | Valid Loss=15.0412 | Train Loss Unweighted=2.5983 | Valid Loss Unweighted=2.2767 | Stale=19 | ETA=669.2m +rank0 [2026-11-15 04:52:33,506] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:52:33,650] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:52:33,655] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:33,706] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:52:34,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:52:34,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:52:34,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:52:34,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:52:34,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:52:35,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:52:35,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:52:35,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:52:35,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:52:35,689] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:52:35,851] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:52:36,016] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:52:36,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:52:36,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:52:36,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:52:36,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:52:36,759] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:52:36,767] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:37,161] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:37,421] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:38,349] INFO : Plotted jets +rank0 [2026-11-15 04:52:39,555] INFO : Plotted jet ratio +rank0 [2026-11-15 04:53:15,954] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:53:15,954] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:55,769] INFO : Step 23000/500000 rank0 | Train Loss: 12.2439 | Train Loss Unweighted: 2.3527 | Train Loss Components: Classification: 0.0121 | Classification_binary: 2.1547 | Regression_cos_phi: 0.0044 | Regression_energy: 0.0866 | Regression_eta: 0.0052 | Regression_pt: 0.0856 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5834 | Classification_binary: 2.1547 | Regression_cos_phi: 1.7519 | Regression_energy: 1.7688 | Regression_eta: 1.7169 | Regression_pt: 1.7407 | Regression_sin_phi: 1.5275 | LR: 4.73e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8425s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3340.8 | uss_MB: 2078.2 | child_rss_MB: 23107.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:53:55,770] INFO : Running validation on rank0 +rank0 [2026-11-15 04:53:55,771] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:53:55,771] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:53:55,877] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:53:55,958] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:53:56,038] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:53:56,088] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:53:56,134] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:53:56,180] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:53:56,228] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:53:56,275] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:53:56,346] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:53:56,425] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:53:56,504] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:53:56,552] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:53:56,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:53:56,654] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:53:56,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:53:56,747] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:53:56,813] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:53:56,817] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:53:56,820] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:53:56,821] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:53:56,822] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:53:56,824] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:53:56,825] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:53:56,827] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:53:56,829] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:53:56,830] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:53:56,832] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:53:56,834] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:53:56,835] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:53:56,837] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:53:56,838] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:53:56,840] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:53:57,792] INFO : VALIDATION | Step=23000/500000 | Train Loss=12.2439 | Valid Loss=15.1833 | Train Loss Unweighted=2.3527 | Valid Loss Unweighted=2.2832 | Stale=20 | ETA=667.9m +rank0 [2026-11-15 04:53:57,792] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:53:57,960] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:53:57,965] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:58,029] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:53:58,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:53:58,729] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:53:58,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:53:59,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:53:59,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:53:59,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:53:59,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:53:59,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:53:59,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:54:00,060] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:54:00,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:54:00,408] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:54:00,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:54:00,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:54:00,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:54:01,052] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:54:01,129] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:54:01,136] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:54:01,517] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:54:01,773] INFO : Plotted number of elements +rank0 [2026-11-15 04:54:03,175] INFO : Plotted jets +rank0 [2026-11-15 04:54:03,898] INFO : Plotted jet ratio +rank0 [2026-11-15 04:54:07,497] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:54:07,497] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:54:50,149] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:54:50,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:20,515] INFO : Step 24000/500000 rank0 | Train Loss: 8.9178 | Train Loss Unweighted: 1.9522 | Train Loss Components: Classification: 0.0126 | Classification_binary: 1.7864 | Regression_cos_phi: 0.0025 | Regression_energy: 0.0742 | Regression_eta: 0.0017 | Regression_pt: 0.0724 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.6418 | Classification_binary: 1.7864 | Regression_cos_phi: 0.9834 | Regression_energy: 1.5159 | Regression_eta: 0.5472 | Regression_pt: 1.4711 | Regression_sin_phi: 0.9721 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8210s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0256 | max_loss: 0.0025 | max_backward: 0.0277 | max_optimizer_step: 0.0897 | max_tensorboard_logging: 0.2655 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3369.3 | uss_MB: 2106.7 | child_rss_MB: 23108.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:55:20,517] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:55:20,567] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:20,567] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:20,567] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:20,673] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:55:20,753] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:20,832] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:20,881] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:20,927] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:20,974] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:21,020] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:21,066] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:21,138] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:21,218] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:21,296] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:21,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:21,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:21,438] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:21,485] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:21,531] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:21,599] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:21,603] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:21,604] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:21,607] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:21,608] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:21,609] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:21,611] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:21,613] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:21,614] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:21,616] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:21,617] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:21,619] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:21,620] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:21,622] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:21,624] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:21,625] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:22,593] INFO : VALIDATION | Step=24000/500000 | Train Loss=8.9178 | Valid Loss=15.2309 | Train Loss Unweighted=1.9522 | Valid Loss Unweighted=2.2872 | Stale=21 | ETA=666.7m +rank0 [2026-11-15 04:55:22,593] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:55:22,938] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:55:22,943] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:23,006] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:23,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:23,735] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:23,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:24,066] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:24,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:24,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:24,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:24,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:24,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:25,064] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:25,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:25,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:25,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:25,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:25,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:26,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:26,114] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:55:26,122] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:26,504] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:26,760] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:27,688] INFO : Plotted jets +rank0 [2026-11-15 04:55:28,884] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:41,477] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:55:41,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:22,988] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:56:22,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:44,055] INFO : Step 25000/500000 rank0 | Train Loss: 10.4613 | Train Loss Unweighted: 1.9660 | Train Loss Components: Classification: 0.0100 | Classification_binary: 1.8029 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0693 | Regression_eta: 0.0039 | Regression_pt: 0.0718 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3040 | Classification_binary: 1.8029 | Regression_cos_phi: 1.6712 | Regression_energy: 1.4145 | Regression_eta: 1.3008 | Regression_pt: 1.4603 | Regression_sin_phi: 1.5078 | LR: 5.22e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8663s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0253 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2960 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3394.3 | uss_MB: 2131.7 | child_rss_MB: 23107.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:56:44,056] INFO : Running validation on rank0 +rank0 [2026-11-15 04:56:44,056] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:56:44,056] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:56:44,161] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:56:44,242] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:56:44,321] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:56:44,370] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:56:44,417] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:56:44,464] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:56:44,512] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:56:44,559] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:56:44,631] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:56:44,709] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:56:44,789] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:56:44,837] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:56:44,882] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:56:44,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:56:44,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:56:45,022] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:56:45,088] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:56:45,092] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:56:45,094] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:56:45,096] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:56:45,101] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:56:45,103] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:56:45,104] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:56:45,106] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:56:45,107] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:56:45,109] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:56:45,111] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:56:45,112] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:56:45,114] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:56:45,115] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:56:45,117] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:56:45,119] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:56:46,064] INFO : VALIDATION | Step=25000/500000 | Train Loss=10.4613 | Valid Loss=15.3322 | Train Loss Unweighted=1.9660 | Valid Loss Unweighted=2.2940 | Stale=22 | ETA=665.1m +rank0 [2026-11-15 04:56:46,065] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:56:46,412] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:56:46,417] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:46,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:56:47,007] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:56:47,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:56:47,336] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:56:47,497] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:56:47,662] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:56:47,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:56:47,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:56:48,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:56:48,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:56:48,490] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:56:48,654] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:56:48,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:56:48,987] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:56:49,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:56:49,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:56:49,467] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:56:49,547] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:56:49,555] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:49,943] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:50,199] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:51,126] INFO : Plotted jets +rank0 [2026-11-15 04:56:52,310] INFO : Plotted jet ratio +rank0 [2026-11-15 04:57:14,594] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:57:14,594] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:57:57,342] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:57:57,342] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:58:09,338] INFO : Step 26000/500000 rank0 | Train Loss: 8.3759 | Train Loss Unweighted: 2.0088 | Train Loss Components: Classification: 0.0122 | Classification_binary: 1.8517 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0693 | Regression_eta: 0.0018 | Regression_pt: 0.0701 | Regression_sin_phi: 0.0014 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5858 | Classification_binary: 1.8517 | Regression_cos_phi: 0.9338 | Regression_energy: 1.4142 | Regression_eta: 0.6049 | Regression_pt: 1.4259 | Regression_sin_phi: 0.5596 | LR: 5.47e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0024 | max_backward: 0.0264 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3420.3 | uss_MB: 2157.8 | child_rss_MB: 23110.0 | allocated_MB: 170.4 | reserved_MB: 666.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:58:09,340] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:58:09,390] INFO : Running validation on rank0 +rank0 [2026-11-15 04:58:09,390] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:58:09,390] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:58:09,503] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:58:09,583] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:58:09,662] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:58:09,712] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:58:09,758] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:58:09,804] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:58:09,851] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:58:09,898] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:58:09,969] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:58:10,049] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:58:10,128] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:58:10,176] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:58:10,223] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:58:10,267] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:58:10,314] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:58:10,359] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:58:10,428] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:58:10,432] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:58:10,434] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:58:10,436] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:58:10,437] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:58:10,439] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:58:10,440] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:58:10,442] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:58:10,443] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:58:10,445] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:58:10,447] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:58:10,448] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:58:10,450] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:58:10,451] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:58:10,453] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:58:10,454] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:58:11,426] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.3759 | Valid Loss=15.2828 | Train Loss Unweighted=2.0088 | Valid Loss Unweighted=2.2940 | Stale=23 | ETA=664.2m +rank0 [2026-11-15 04:58:11,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:58:11,548] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:58:11,554] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:11,616] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:58:12,114] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:58:12,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:58:12,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:58:12,606] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:58:12,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:58:12,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:58:13,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:58:13,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:58:13,437] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:58:13,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:58:13,769] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:58:13,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:58:14,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:58:14,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:58:14,429] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:58:14,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:58:14,667] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:58:14,677] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:15,063] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:15,320] INFO : Plotted number of elements +rank0 [2026-11-15 04:58:16,254] INFO : Plotted jets +rank0 [2026-11-15 04:58:17,448] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:48,351] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 04:58:48,351] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:30,350] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 04:59:30,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:32,912] INFO : Step 27000/500000 rank0 | Train Loss: 10.3843 | Train Loss Unweighted: 1.9298 | Train Loss Components: Classification: 0.0106 | Classification_binary: 1.7659 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0697 | Regression_eta: 0.0046 | Regression_pt: 0.0717 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.3860 | Classification_binary: 1.7659 | Regression_cos_phi: 1.0365 | Regression_energy: 1.4233 | Regression_eta: 1.5109 | Regression_pt: 1.4586 | Regression_sin_phi: 1.8032 | LR: 5.74e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8398s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0024 | max_backward: 0.0275 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3450.5 | uss_MB: 2187.9 | child_rss_MB: 23108.5 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:59:32,914] INFO : Running validation on rank0 +rank0 [2026-11-15 04:59:32,914] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:59:32,914] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:59:33,017] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 04:59:33,097] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:59:33,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:59:33,229] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:59:33,277] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:59:33,324] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:59:33,371] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:59:33,417] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:59:33,489] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:59:33,569] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:59:33,649] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:59:33,697] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:59:33,743] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:59:33,789] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:59:33,836] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:59:33,882] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:59:33,953] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:59:33,958] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:59:33,961] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:59:33,962] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:59:33,963] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:59:33,964] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:59:33,966] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:59:33,968] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:59:33,970] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:59:33,972] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:59:33,973] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:59:33,974] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:59:33,976] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:59:33,977] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:59:33,979] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:59:33,980] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:59:34,950] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.3843 | Valid Loss=15.6013 | Train Loss Unweighted=1.9298 | Valid Loss Unweighted=2.3034 | Stale=24 | ETA=662.6m +rank0 [2026-11-15 04:59:34,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:59:35,079] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:59:35,084] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:35,147] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:59:35,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:59:35,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:59:36,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:59:36,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:59:36,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:59:36,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:59:36,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:59:36,835] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:59:37,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:59:37,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:59:37,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:59:37,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:59:37,675] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:59:37,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:59:38,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:59:38,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:59:38,239] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:59:38,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:38,625] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:38,881] INFO : Plotted number of elements +rank0 [2026-11-15 04:59:39,812] INFO : Plotted jets +rank0 [2026-11-15 04:59:41,013] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:22,196] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 05:00:22,197] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:57,854] INFO : Step 28000/500000 rank0 | Train Loss: 9.0165 | Train Loss Unweighted: 2.1042 | Train Loss Components: Classification: 0.0115 | Classification_binary: 1.9649 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0590 | Regression_eta: 0.0025 | Regression_pt: 0.0604 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5024 | Classification_binary: 1.9649 | Regression_cos_phi: 1.4890 | Regression_energy: 1.2046 | Regression_eta: 0.8243 | Regression_pt: 1.2286 | Regression_sin_phi: 0.8027 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0025 | max_backward: 0.0265 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2509 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3470.6 | uss_MB: 2208.0 | child_rss_MB: 23110.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:00:57,856] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 05:00:57,907] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:57,907] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:57,907] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:58,012] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 05:00:58,094] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:58,175] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:58,226] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:58,272] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:58,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:58,365] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:58,413] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:58,486] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:58,571] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:58,650] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:58,699] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:58,746] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:58,793] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:58,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:58,890] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:58,958] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:58,963] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:58,964] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:58,966] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:58,968] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:58,969] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:58,971] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:58,972] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:58,974] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:58,975] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:58,977] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:58,979] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:58,980] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:58,982] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:58,983] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:58,985] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:59,955] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.0165 | Valid Loss=15.5100 | Train Loss Unweighted=2.1042 | Valid Loss Unweighted=2.3018 | Stale=25 | ETA=661.5m +rank0 [2026-11-15 05:00:59,956] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:01:00,295] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:01:00,300] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:00,364] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:01:00,930] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:01:01,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:01:01,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:01:01,430] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:01:01,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:01:01,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:01:01,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:01:02,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:01:02,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:01:02,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:01:02,595] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:01:02,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:01:02,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:01:03,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:01:03,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:01:03,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:01:03,522] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:01:03,530] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:03,930] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:01:04,189] INFO : Plotted number of elements +rank0 [2026-11-15 05:01:05,172] INFO : Plotted jets +rank0 [2026-11-15 05:01:06,386] INFO : Plotted jet ratio +rank0 [2026-11-15 05:01:14,174] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 05:01:14,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:01:55,837] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 05:01:55,837] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:02:21,617] INFO : Step 29000/500000 rank0 | Train Loss: 10.4576 | Train Loss Unweighted: 1.7359 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.5747 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0682 | Regression_eta: 0.0057 | Regression_pt: 0.0693 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.4260 | Classification_binary: 1.5747 | Regression_cos_phi: 1.2835 | Regression_energy: 1.3927 | Regression_eta: 1.8902 | Regression_pt: 1.4083 | Regression_sin_phi: 1.4823 | LR: 6.29e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0283 | max_loss: 0.0026 | max_backward: 0.0274 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2681 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3492.5 | uss_MB: 2230.0 | child_rss_MB: 23108.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:02:21,619] INFO : Running validation on rank0 +rank0 [2026-11-15 05:02:21,619] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:02:21,619] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:02:21,724] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 05:02:21,803] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:02:21,883] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:02:21,933] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:02:21,980] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:02:22,026] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:02:22,074] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:02:22,121] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:02:22,194] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:02:22,273] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:02:22,351] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:02:22,400] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:02:22,446] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:02:22,492] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:02:22,538] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:02:22,584] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:02:22,652] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:02:22,657] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:02:22,659] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:02:22,660] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:02:22,662] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:02:22,664] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:02:22,665] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:02:22,667] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:02:22,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:02:22,670] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:02:22,672] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:02:22,673] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:02:22,675] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:02:22,676] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:02:22,678] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:02:22,680] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:02:23,629] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.4576 | Valid Loss=15.5704 | Train Loss Unweighted=1.7359 | Valid Loss Unweighted=2.3133 | Stale=26 | ETA=660.0m +rank0 [2026-11-15 05:02:23,629] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:02:23,781] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:02:23,786] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:23,837] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:02:24,302] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:02:24,473] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:02:24,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:02:24,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:02:24,966] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:02:25,131] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:02:25,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:02:25,457] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:02:25,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:02:25,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:02:25,948] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:02:26,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:02:26,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:02:26,454] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:02:26,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:02:26,768] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:02:26,877] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:02:26,884] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:27,275] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:27,529] INFO : Plotted number of elements +rank0 [2026-11-15 05:02:28,492] INFO : Plotted jets +rank0 [2026-11-15 05:02:29,697] INFO : Plotted jet ratio +rank0 [2026-11-15 05:02:47,186] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 05:02:47,186] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:29,810] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 05:03:29,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:46,740] INFO : Step 30000/500000 rank0 | Train Loss: 9.5773 | Train Loss Unweighted: 2.0052 | Train Loss Components: Classification: 0.0118 | Classification_binary: 1.8595 | Regression_cos_phi: 0.0029 | Regression_energy: 0.0615 | Regression_eta: 0.0052 | Regression_pt: 0.0622 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5433 | Classification_binary: 1.8595 | Regression_cos_phi: 1.1315 | Regression_energy: 1.2561 | Regression_eta: 1.7376 | Regression_pt: 1.2654 | Regression_sin_phi: 0.7840 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8262s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0249 | max_loss: 0.0026 | max_backward: 0.0275 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2671 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3512.7 | uss_MB: 2250.2 | child_rss_MB: 23108.2 | allocated_MB: 170.4 | reserved_MB: 664.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:03:46,742] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 05:03:46,791] INFO : Running validation on rank0 +rank0 [2026-11-15 05:03:46,791] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:03:46,791] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:03:46,919] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 05:03:47,003] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:03:47,083] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:03:47,133] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:03:47,179] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:03:47,225] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:03:47,272] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:03:47,319] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:03:47,390] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:03:47,470] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:03:47,547] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:03:47,595] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:03:47,642] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:03:47,688] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:03:47,734] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:03:47,780] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:03:47,847] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:03:47,852] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:03:47,854] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:03:47,855] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:03:47,857] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:03:47,859] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:03:47,860] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:03:47,862] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:03:47,863] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:03:47,865] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:03:47,866] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:03:47,868] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:03:47,870] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:03:47,871] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:03:47,873] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:03:47,874] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:03:48,841] INFO : VALIDATION | Step=30000/500000 | Train Loss=9.5773 | Valid Loss=15.8482 | Train Loss Unweighted=2.0052 | Valid Loss Unweighted=2.3270 | Stale=27 | ETA=658.8m +rank0 [2026-11-15 05:03:48,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:03:48,965] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:03:48,970] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:49,038] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:03:49,565] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:03:49,737] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:03:49,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:03:50,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:03:50,227] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:03:50,388] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:03:50,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:03:50,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:03:50,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:03:51,056] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:03:51,232] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:03:51,397] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:03:51,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:03:51,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:03:51,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:03:52,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:03:52,104] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:03:52,112] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:52,499] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:52,754] INFO : Plotted number of elements +rank0 [2026-11-15 05:03:53,679] INFO : Plotted jets +rank0 [2026-11-15 05:03:54,893] INFO : Plotted jet ratio +rank0 [2026-11-15 05:04:21,570] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 05:04:21,570] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:05:04,054] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 05:05:04,055] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:05:11,638] INFO : Step 31000/500000 rank0 | Train Loss: 12.6850 | Train Loss Unweighted: 2.1436 | Train Loss Components: Classification: 0.0151 | Classification_binary: 1.9460 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0836 | Regression_eta: 0.0081 | Regression_pt: 0.0840 | Regression_sin_phi: 0.0037 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.9664 | Classification_binary: 1.9460 | Regression_cos_phi: 1.2331 | Regression_energy: 1.7075 | Regression_eta: 2.6865 | Regression_pt: 1.7071 | Regression_sin_phi: 1.4385 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8818s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.3208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3458.8 | uss_MB: 2196.2 | child_rss_MB: 23102.4 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 05:05:11,640] INFO : Running validation on rank0 +rank0 [2026-11-15 05:05:11,640] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:05:11,640] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:05:11,745] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 05:05:11,826] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:05:11,905] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:05:11,956] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:05:12,003] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:05:12,051] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:05:12,099] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:05:12,146] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:05:12,218] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:05:12,306] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:05:12,384] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:05:12,432] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:05:12,481] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:05:12,528] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:05:12,576] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:05:12,625] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:05:12,695] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:05:12,699] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:05:12,701] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:05:12,703] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:05:12,705] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:05:12,706] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:05:12,709] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:05:12,709] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:05:12,711] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:05:12,714] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:05:12,714] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:05:12,717] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:05:12,717] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:05:12,719] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:05:12,721] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:05:12,722] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:05:13,717] INFO : VALIDATION | Step=31000/500000 | Train Loss=12.6850 | Valid Loss=15.8210 | Train Loss Unweighted=2.1436 | Valid Loss Unweighted=2.3241 | Stale=28 | ETA=657.6m +rank0 [2026-11-15 05:05:13,717] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:05:14,053] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:05:14,058] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:14,110] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:05:14,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:05:14,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:05:14,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:05:15,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:05:15,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:05:15,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:05:15,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:05:15,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:05:15,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:05:16,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:05:16,266] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:05:16,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:05:16,604] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:05:16,767] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:05:16,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:05:17,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:17,209] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:05:17,216] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,623] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:17,882] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:18,825] INFO : Plotted jets +rank0 [2026-11-15 05:05:20,108] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:56,054] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-15 05:05:56,054] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:36,900] INFO : Step 32000/500000 rank0 | Train Loss: 12.2766 | Train Loss Unweighted: 2.4380 | Train Loss Components: Classification: 0.0118 | Classification_binary: 2.2403 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0862 | Regression_eta: 0.0034 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.5345 | Classification_binary: 2.2403 | Regression_cos_phi: 1.4572 | Regression_energy: 1.7611 | Regression_eta: 1.1201 | Regression_pt: 1.7568 | Regression_sin_phi: 2.4066 | LR: 7.19e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.9474s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0026 | max_backward: 0.0282 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.3953 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3568.9 | uss_MB: 2370.9 | child_rss_MB: 23107.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:06:36,902] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/checkpoints/checkpoint-32000.pth +rank0 [2026-11-15 05:06:36,951] INFO : Running validation on rank0 +rank0 [2026-11-15 05:06:36,951] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:06:36,951] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:06:37,054] INFO : saved batch0_epoch32000.parquet +rank0 [2026-11-15 05:06:37,133] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:06:37,212] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:06:37,261] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:06:37,306] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:06:37,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:06:37,399] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:06:37,445] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:06:37,517] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:06:37,595] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:06:37,675] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:06:37,722] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:06:37,768] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:06:37,813] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:06:37,861] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:06:37,908] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:06:37,976] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:06:37,980] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:06:37,981] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:06:37,983] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:06:37,985] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:06:37,986] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:06:37,988] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:06:37,989] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:06:37,991] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:06:37,992] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:06:37,994] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:06:37,996] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:06:37,997] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:06:37,999] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:06:38,000] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:06:38,002] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:06:38,950] INFO : VALIDATION | Step=32000/500000 | Train Loss=12.2766 | Valid Loss=15.9270 | Train Loss Unweighted=2.4380 | Valid Loss Unweighted=2.3375 | Stale=29 | ETA=656.5m +rank0 [2026-11-15 05:06:38,950] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:06:39,269] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:06:39,274] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:39,326] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:06:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:06:39,970] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:06:40,133] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:06:40,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:06:40,474] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:06:40,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:06:40,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:06:40,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:06:41,139] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:06:41,305] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:06:41,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:06:41,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:06:41,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:06:41,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:06:42,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:06:42,307] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:06:42,407] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:06:42,414] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:42,819] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_32000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:43,076] INFO : Plotted number of elements +rank0 [2026-11-15 05:06:44,037] INFO : Plotted jets +rank0 [2026-11-15 05:06:45,240] INFO : Plotted jet ratio +rank0 [2026-11-15 05:06:48,206] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-15 05:06:48,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:07:30,456] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-15 05:07:30,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:08:01,599] INFO : Step 33000/500000 rank0 | Train Loss: 7.2653 | Train Loss Unweighted: 1.8358 | Train Loss Components: Classification: 0.0087 | Classification_binary: 1.7041 | Regression_cos_phi: 0.0018 | Regression_energy: 0.0587 | Regression_eta: 0.0021 | Regression_pt: 0.0588 | Regression_sin_phi: 0.0016 | Task Weights: Classification: 130.4132 | Classification_binary: 1.0000 | Regression_cos_phi: 395.7253 | Regression_energy: 20.4205 | Regression_eta: 331.0066 | Regression_pt: 20.3297 | Regression_sin_phi: 388.8638 | Task Weighted Losses: Classification: 1.1341 | Classification_binary: 1.7041 | Regression_cos_phi: 0.6959 | Regression_energy: 1.1990 | Regression_eta: 0.7036 | Regression_pt: 1.1952 | Regression_sin_phi: 0.6334 | LR: 7.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8713s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0912 | max_tensorboard_logging: 0.3202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3593.3 | uss_MB: 2395.3 | child_rss_MB: 23107.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:08:01,600] INFO : Running validation on rank0 +rank0 [2026-11-15 05:08:01,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:08:01,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:08:01,704] INFO : saved batch0_epoch33000.parquet +rank0 [2026-11-15 05:08:01,782] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:08:01,861] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:08:01,911] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:08:01,957] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:08:02,003] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:08:02,050] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:08:02,097] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:08:02,168] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:08:02,246] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:08:02,325] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:08:02,371] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:08:02,418] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:08:02,464] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:08:02,510] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:08:02,556] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:08:02,624] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:08:02,628] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:08:02,630] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:08:02,633] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:08:02,634] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:08:02,636] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:08:02,637] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:08:02,639] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:08:02,641] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:08:02,642] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:08:02,644] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:08:02,646] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:08:02,647] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:08:02,653] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:08:02,655] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:08:02,657] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:08:03,610] INFO : VALIDATION | Step=33000/500000 | Train Loss=7.2653 | Valid Loss=16.1339 | Train Loss Unweighted=1.8358 | Valid Loss Unweighted=2.3402 | Stale=30 | ETA=655.2m +rank0 [2026-11-15 05:08:03,611] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:08:03,787] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:08:03,796] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:03,847] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:08:04,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:08:04,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:08:04,646] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:08:04,810] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:08:04,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:08:05,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:08:05,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:08:05,477] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:08:05,652] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:08:05,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:08:05,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:08:06,138] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:08:06,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:08:06,484] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:08:06,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:08:06,811] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:08:06,892] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:08:06,900] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:07,296] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_25mm_seed12345_20261115_042136_106951/preds_step_33000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:08:07,552] INFO : Plotted number of elements +rank0 [2026-11-15 05:08:08,482] INFO : Plotted jets +rank0 [2026-11-15 05:08:09,689] INFO : Plotted jet ratio +rank0 [2026-11-15 05:08:10,083] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 05:08:10,083] INFO : Training completed. Total time on device 0: 46.408min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..eae38769053669f435d8113a1772c4a18bdc5ddd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f5222ed4a01c7c3e69143b260cd2eb48ea291d87b279f8544090cf759149ee7 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-30000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-30000.pth new file mode 100644 index 0000000000000000000000000000000000000000..33957a35ceeb7f7d9c86a2b2f57ca197a51290da --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-30000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42214eef6cff7041288be51ef05e012c7c456d72e1792b978c4f094a8b916ef6 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..ed0cfd927d04c53774709375a9108deb2b406647 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 1.7793594306049823, "train": {"Classification_binary": 2.070162057876587, "Classification": 0.01604004018008709, "Regression_pt": 0.12082915008068085, "Regression_eta": 0.0035314401611685753, "Regression_sin_phi": 0.005307347513735294, "Regression_cos_phi": 0.005610913969576359, "Regression_energy": 0.11989401280879974, "Total": 14.138497352600098, "Total_unweighted": 2.341374962590635}, "train_group": {}, "valid": {"Classification_binary": 1.97477126121521, "Classification": 0.015558799728751183, "Regression_pt": 0.0980813130736351, "Regression_eta": 0.00577414408326149, "Regression_sin_phi": 0.005804380401968956, "Regression_cos_phi": 0.006291543133556843, "Regression_energy": 0.09619550406932831, "Total": 14.180630683898926, "Total_unweighted": 2.202476945705712, "metrics/particle/class_1/efficiency": 0.9468299150466919, "metrics/particle/class_1/pid_accuracy": 0.9865151643753052, "metrics/particle/class_2/efficiency": 0.5490654110908508, "metrics/particle/class_2/pid_accuracy": 0.8212766051292419, "metrics/particle/class_3/efficiency": 0.8346891403198242, "metrics/particle/class_3/pid_accuracy": 0.9892098903656006, "metrics/particle/class_4/efficiency": 0.899350643157959, "metrics/particle/class_4/pid_accuracy": 0.5451263785362244, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7514792680740356, "metrics/particle/count/bias_mean": 0.3319999873638153, "metrics/particle/count/mae": 2.9800000190734863, "metrics/particle/count/prediction_mean": 96.14399719238281, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026880202814936638, "metrics/particle/event/energy_response_mean": 1.004536509513855, "metrics/particle/event/met_abs_error": 4.151976585388184, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0256495364010334, "metrics/particle/event/scalar_pt_response_mean": 1.0067095756530762, "metrics/particle/event/vector_pt_closure": 0.02664298750460148, "metrics/particle/matched/delta_eta_abs_mean": 0.008675780147314072, "metrics/particle/matched/delta_phi_abs_mean": 0.009607796557247639, "metrics/particle/matched/delta_r_mean": 0.014481591060757637, "metrics/particle/matched/energy_relative_abs_mean": 0.08654677122831345, "metrics/particle/matched/pid_accuracy": 0.9723982214927673, "metrics/particle/matched/pt_relative_abs_mean": 0.08132191002368927, "metrics/particle/matching/duplicate_fraction": 0.026044266298413277, "metrics/particle/matching/efficiency": 0.8697031736373901, "metrics/particle/matching/f1": 0.8681989908218384, "metrics/particle/matching/purity": 0.8666999340057373, "metrics/particle/matching/target_coverage_dr0p05": 0.8491212129592896, "metrics/particle/matching/target_coverage_dr0p10": 0.8990940451622009, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7667098045349121, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8983426094055176}, "step_train_time": 0.7687821388244629, "step_valid_time": 1.8250611050170846, "step_total_time": 76.92167520523071, "elapsed_training_loop_seconds": 76.92167520523071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042132874602814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058099039347008885, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1445, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047337278106508875, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9526627218934911, "step_test_time": 3.6921777140232734, "step_plot_time": 2.75459280598443, "step_validation_cycle_time": 8.672396269976161} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..6c926f2f3d4735ef7934e3bfde4a91cac4b75443 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 1.6080416440963745, "Classification": 0.012921852059662342, "Regression_pt": 0.0707707479596138, "Regression_eta": 0.004073502495884895, "Regression_sin_phi": 0.004023971036076546, "Regression_cos_phi": 0.0036713997833430767, "Regression_energy": 0.07247956842184067, "Total": 10.299359321594238, "Total_unweighted": 1.7759826858527958}, "train_group": {}, "valid": {"Classification_binary": 1.9996603727340698, "Classification": 0.016071254387497902, "Regression_pt": 0.09899109601974487, "Regression_eta": 0.005818702280521393, "Regression_sin_phi": 0.005843994207680225, "Regression_cos_phi": 0.0063523417338728905, "Regression_energy": 0.0973154827952385, "Total": 14.364747047424316, "Total_unweighted": 2.2300532441586256, "metrics/particle/class_1/efficiency": 0.950677752494812, "metrics/particle/class_1/pid_accuracy": 0.9865697622299194, "metrics/particle/class_2/efficiency": 0.5514018535614014, "metrics/particle/class_2/pid_accuracy": 0.8262711763381958, "metrics/particle/class_3/efficiency": 0.8334301710128784, "metrics/particle/class_3/pid_accuracy": 0.9893097877502441, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.5615941882133484, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7588235139846802, "metrics/particle/count/bias_mean": 0.30399999022483826, "metrics/particle/count/mae": 3.072000026702881, "metrics/particle/count/prediction_mean": 96.11599731445312, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027795935049653053, "metrics/particle/event/energy_response_mean": 0.9997712969779968, "metrics/particle/event/met_abs_error": 4.1794352531433105, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02707570791244507, "metrics/particle/event/scalar_pt_response_mean": 1.002077341079712, "metrics/particle/event/vector_pt_closure": 0.02845207415521145, "metrics/particle/matched/delta_eta_abs_mean": 0.008511972613632679, "metrics/particle/matched/delta_phi_abs_mean": 0.009651214815676212, "metrics/particle/matched/delta_r_mean": 0.014388985000550747, "metrics/particle/matched/energy_relative_abs_mean": 0.08660097420215607, "metrics/particle/matched/pid_accuracy": 0.9729716777801514, "metrics/particle/matched/pt_relative_abs_mean": 0.0810886025428772, "metrics/particle/matching/duplicate_fraction": 0.026759332045912743, "metrics/particle/matching/efficiency": 0.8711643815040588, "metrics/particle/matching/f1": 0.8697844743728638, "metrics/particle/matching/purity": 0.8684089779853821, "metrics/particle/matching/target_coverage_dr0p05": 0.8510416150093079, "metrics/particle/matching/target_coverage_dr0p10": 0.899469792842865, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7675447463989258, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8986765742301941}, "step_train_time": 0.7334444522857666, "step_valid_time": 1.9964512499864213, "step_total_time": 817.7534351348877, "elapsed_training_loop_seconds": 817.7534351348877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991716555239976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06354192466278075, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9539776462853385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9552337063857801, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546052631578947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04476629361421988, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520052596975674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9532587228439763, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526315789473684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9539776462853385, "step_test_time": 3.128889749990776, "step_plot_time": 2.73142398300115, "step_validation_cycle_time": 8.20755025401013} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..f088536d5f3fbed863512eea545ff319452a02bf --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 19.572953736654803, "train": {"Classification_binary": 2.126652717590332, "Classification": 0.012210331857204437, "Regression_pt": 0.10925601422786713, "Regression_eta": 0.02634519711136818, "Regression_sin_phi": 0.005846972577273846, "Regression_cos_phi": 0.0059452960267663, "Regression_energy": 0.1424577385187149, "Total": 21.58833122253418, "Total_unweighted": 2.428714267909527}, "train_group": {}, "valid": {"Classification_binary": 2.010101556777954, "Classification": 0.01613941602408886, "Regression_pt": 0.09926905483007431, "Regression_eta": 0.005887026898562908, "Regression_sin_phi": 0.005867225117981434, "Regression_cos_phi": 0.006373928859829903, "Regression_energy": 0.09746291488409042, "Total": 14.430913925170898, "Total_unweighted": 2.241101123392582, "metrics/particle/class_1/efficiency": 0.9484040141105652, "metrics/particle/class_1/pid_accuracy": 0.9869986176490784, "metrics/particle/class_2/efficiency": 0.5443925261497498, "metrics/particle/class_2/pid_accuracy": 0.8218883872032166, "metrics/particle/class_3/efficiency": 0.8331396579742432, "metrics/particle/class_3/pid_accuracy": 0.9893060326576233, "metrics/particle/class_4/efficiency": 0.8863636255264282, "metrics/particle/class_4/pid_accuracy": 0.5274725556373596, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7751479148864746, "metrics/particle/count/bias_mean": 0.035999998450279236, "metrics/particle/count/mae": 3.0280001163482666, "metrics/particle/count/prediction_mean": 95.8479995727539, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02634553238749504, "metrics/particle/event/energy_response_mean": 0.9984091520309448, "metrics/particle/event/met_abs_error": 4.239970684051514, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026174193248152733, "metrics/particle/event/scalar_pt_response_mean": 1.0004864931106567, "metrics/particle/event/vector_pt_closure": 0.027416344732046127, "metrics/particle/matched/delta_eta_abs_mean": 0.008549266494810581, "metrics/particle/matched/delta_phi_abs_mean": 0.00976896844804287, "metrics/particle/matched/delta_r_mean": 0.014501860365271568, "metrics/particle/matched/energy_relative_abs_mean": 0.08664634078741074, "metrics/particle/matched/pid_accuracy": 0.9728172421455383, "metrics/particle/matched/pt_relative_abs_mean": 0.081212617456913, "metrics/particle/matching/duplicate_fraction": 0.02604123204946518, "metrics/particle/matching/efficiency": 0.8692857027053833, "metrics/particle/matching/f1": 0.8691223859786987, "metrics/particle/matching/purity": 0.8689591884613037, "metrics/particle/matching/target_coverage_dr0p05": 0.8482444882392883, "metrics/particle/matching/target_coverage_dr0p10": 0.8983426094055176, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7660835981369019, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8980085849761963}, "step_train_time": 0.8287477493286133, "step_valid_time": 1.9993336519692093, "step_total_time": 899.526917219162, "elapsed_training_loop_seconds": 899.526917219162, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999740432649732, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06417397757274879, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559789750328516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9562931317778508, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04402102496714849, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540078843626807, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.954321393361814, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566074950690335, "step_test_time": 3.310481714026537, "step_plot_time": 2.746889811998699, "step_validation_cycle_time": 8.412877233990002} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..912c2b7b5296620f8da38c0209edff87d0bc8642 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 21.352313167259787, "train": {"Classification_binary": 1.8560361862182617, "Classification": 0.010925265029072762, "Regression_pt": 0.07030011713504791, "Regression_eta": 0.003626464167609811, "Regression_sin_phi": 0.003484422340989113, "Regression_cos_phi": 0.004293049219995737, "Regression_energy": 0.07518676668405533, "Total": 10.218034744262695, "Total_unweighted": 2.0238522707950324}, "train_group": {}, "valid": {"Classification_binary": 2.0152013301849365, "Classification": 0.016265835613012314, "Regression_pt": 0.09996157139539719, "Regression_eta": 0.006007818970829248, "Regression_sin_phi": 0.005915760528296232, "Regression_cos_phi": 0.00647305091843009, "Regression_energy": 0.09828778356313705, "Total": 14.575593948364258, "Total_unweighted": 2.2481131511740386, "metrics/particle/class_1/efficiency": 0.9501530528068542, "metrics/particle/class_1/pid_accuracy": 0.9872986674308777, "metrics/particle/class_2/efficiency": 0.5391355156898499, "metrics/particle/class_2/pid_accuracy": 0.8179848194122314, "metrics/particle/class_3/efficiency": 0.8312028050422668, "metrics/particle/class_3/pid_accuracy": 0.9885820746421814, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.52173912525177, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7602339386940002, "metrics/particle/count/bias_mean": -0.12800000607967377, "metrics/particle/count/mae": 3.0480000972747803, "metrics/particle/count/prediction_mean": 95.68399810791016, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027792755514383316, "metrics/particle/event/energy_response_mean": 0.9979434609413147, "metrics/particle/event/met_abs_error": 4.43477201461792, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02743346244096756, "metrics/particle/event/scalar_pt_response_mean": 1.0005170106887817, "metrics/particle/event/vector_pt_closure": 0.029482537880539894, "metrics/particle/matched/delta_eta_abs_mean": 0.008679059334099293, "metrics/particle/matched/delta_phi_abs_mean": 0.00979621335864067, "metrics/particle/matched/delta_r_mean": 0.014628442004323006, "metrics/particle/matched/energy_relative_abs_mean": 0.08714699000120163, "metrics/particle/matched/pid_accuracy": 0.9722836017608643, "metrics/particle/matched/pt_relative_abs_mean": 0.08152520656585693, "metrics/particle/matching/duplicate_fraction": 0.02646210417151451, "metrics/particle/matching/efficiency": 0.8691186904907227, "metrics/particle/matching/f1": 0.8696996569633484, "metrics/particle/matching/purity": 0.8702813386917114, "metrics/particle/matching/target_coverage_dr0p05": 0.845948338508606, "metrics/particle/matching/target_coverage_dr0p10": 0.8980503678321838, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7653321027755737, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8966308832168579}, "step_train_time": 0.7498607635498047, "step_valid_time": 1.9685662380070426, "step_total_time": 981.1146330833435, "elapsed_training_loop_seconds": 981.1146330833435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007168810269822, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06441492987596165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585798816568047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573210768220617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9579500657030223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04267892317793828, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553512803676953, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9559789750328516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585798816568047, "step_test_time": 3.128457833023276, "step_plot_time": 2.7350517860031687, "step_validation_cycle_time": 8.194001929019578} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..4eab3dcf5e35219d55c041d8ff04cafdfc1d66f0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 23.131672597864767, "train": {"Classification_binary": 1.8362345695495605, "Classification": 0.014382248744368553, "Regression_pt": 0.11495256423950195, "Regression_eta": 0.0044301352463662624, "Regression_sin_phi": 0.0037518416065722704, "Regression_cos_phi": 0.0037611722946166992, "Regression_energy": 0.11281393468379974, "Total": 12.489267349243164, "Total_unweighted": 2.090326466364786}, "train_group": {}, "valid": {"Classification_binary": 2.0089826583862305, "Classification": 0.016270054504275322, "Regression_pt": 0.09921436011791229, "Regression_eta": 0.00608360068872571, "Regression_sin_phi": 0.005932302214205265, "Regression_cos_phi": 0.006411566864699125, "Regression_energy": 0.09760034829378128, "Total": 14.548900604248047, "Total_unweighted": 2.2404948910698295, "metrics/particle/class_1/efficiency": 0.948753833770752, "metrics/particle/class_1/pid_accuracy": 0.9868190884590149, "metrics/particle/class_2/efficiency": 0.5473130941390991, "metrics/particle/class_2/pid_accuracy": 0.8185698986053467, "metrics/particle/class_3/efficiency": 0.830040693283081, "metrics/particle/class_3/pid_accuracy": 0.9887994527816772, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.5144927501678467, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7882353067398071, "metrics/particle/count/bias_mean": 0.2639999985694885, "metrics/particle/count/mae": 2.9679999351501465, "metrics/particle/count/prediction_mean": 96.07599639892578, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026899978518486023, "metrics/particle/event/energy_response_mean": 1.0017439126968384, "metrics/particle/event/met_abs_error": 4.240945816040039, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026077523827552795, "metrics/particle/event/scalar_pt_response_mean": 1.0026650428771973, "metrics/particle/event/vector_pt_closure": 0.028965773060917854, "metrics/particle/matched/delta_eta_abs_mean": 0.00846849288791418, "metrics/particle/matched/delta_phi_abs_mean": 0.009912310168147087, "metrics/particle/matched/delta_r_mean": 0.014552520588040352, "metrics/particle/matched/energy_relative_abs_mean": 0.08707180619239807, "metrics/particle/matched/pid_accuracy": 0.9721674919128418, "metrics/particle/matched/pt_relative_abs_mean": 0.08145571500062943, "metrics/particle/matching/duplicate_fraction": 0.02668720670044422, "metrics/particle/matching/efficiency": 0.8684924840927124, "metrics/particle/matching/f1": 0.8672975897789001, "metrics/particle/matching/purity": 0.8661059737205505, "metrics/particle/matching/target_coverage_dr0p05": 0.8468250036239624, "metrics/particle/matching/target_coverage_dr0p10": 0.8975911140441895, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7649146318435669, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8983426094055176}, "step_train_time": 0.814521312713623, "step_valid_time": 1.9918638940434903, "step_total_time": 1064.152540922165, "elapsed_training_loop_seconds": 1064.152540922165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009181276211874, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06436410637928114, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9559500328731098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553219448094612, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556358856391719, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04467805519053877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539776462853385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533508541392904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536641472231351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9559500328731098, "step_test_time": 3.3470741159981117, "step_plot_time": 2.751581730961334, "step_validation_cycle_time": 8.4451321529923} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..4bfc7eb7941fe0ab94d71392ee07f9262ee607fe --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 24.91103202846975, "train": {"Classification_binary": 2.0703489780426025, "Classification": 0.015367059968411922, "Regression_pt": 0.09288676083087921, "Regression_eta": 0.0024424155708402395, "Regression_sin_phi": 0.003157317638397217, "Regression_cos_phi": 0.004203091375529766, "Regression_energy": 0.09148287028074265, "Total": 11.276213645935059, "Total_unweighted": 2.2798884937074035}, "train_group": {}, "valid": {"Classification_binary": 2.0187814235687256, "Classification": 0.01635739766061306, "Regression_pt": 0.09991515427827835, "Regression_eta": 0.006087227258831263, "Regression_sin_phi": 0.005993004888296127, "Regression_cos_phi": 0.006525845266878605, "Regression_energy": 0.09813997894525528, "Total": 14.659499168395996, "Total_unweighted": 2.2518000318668783, "metrics/particle/class_1/efficiency": 0.9501530528068542, "metrics/particle/class_1/pid_accuracy": 0.9876668453216553, "metrics/particle/class_2/efficiency": 0.5467289686203003, "metrics/particle/class_2/pid_accuracy": 0.8205128312110901, "metrics/particle/class_3/efficiency": 0.8302343487739563, "metrics/particle/class_3/pid_accuracy": 0.9902017712593079, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.5144927501678467, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7633135914802551, "metrics/particle/count/bias_mean": 0.20800000429153442, "metrics/particle/count/mae": 3.0239999294281006, "metrics/particle/count/prediction_mean": 96.0199966430664, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026802368462085724, "metrics/particle/event/energy_response_mean": 0.9999186992645264, "metrics/particle/event/met_abs_error": 4.2151875495910645, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025823276489973068, "metrics/particle/event/scalar_pt_response_mean": 1.0009698867797852, "metrics/particle/event/vector_pt_closure": 0.02851741947233677, "metrics/particle/matched/delta_eta_abs_mean": 0.00850225705653429, "metrics/particle/matched/delta_phi_abs_mean": 0.00978607777506113, "metrics/particle/matched/delta_r_mean": 0.014483439736068249, "metrics/particle/matched/energy_relative_abs_mean": 0.08686729520559311, "metrics/particle/matched/pid_accuracy": 0.9731014966964722, "metrics/particle/matched/pt_relative_abs_mean": 0.08131081610918045, "metrics/particle/matching/duplicate_fraction": 0.026619454845786095, "metrics/particle/matching/efficiency": 0.8691604137420654, "metrics/particle/matching/f1": 0.8682180047035217, "metrics/particle/matching/purity": 0.8672776222229004, "metrics/particle/matching/target_coverage_dr0p05": 0.8487871885299683, "metrics/particle/matching/target_coverage_dr0p10": 0.8980085849761963, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7649146318435669, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8968814015388489}, "step_train_time": 0.766167163848877, "step_valid_time": 2.0042954299715348, "step_total_time": 1147.3018140792847, "elapsed_training_loop_seconds": 1147.3018140792847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992632286467982, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06467947180434286, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04339250493096647, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566074950690335, "step_test_time": 3.1066920830053277, "step_plot_time": 2.736310232023243, "step_validation_cycle_time": 8.206891383975744} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..79a738d71284a6e42fb48abb6d9533fe1719e786 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 26.690391459074732, "train": {"Classification_binary": 1.7426989078521729, "Classification": 0.012509357184171677, "Regression_pt": 0.0850655660033226, "Regression_eta": 0.0036395955830812454, "Regression_sin_phi": 0.0053112986497581005, "Regression_cos_phi": 0.005541249178349972, "Regression_energy": 0.08358556032180786, "Total": 11.890775680541992, "Total_unweighted": 1.9383515347726643}, "train_group": {}, "valid": {"Classification_binary": 2.021930694580078, "Classification": 0.01634107530117035, "Regression_pt": 0.10036318004131317, "Regression_eta": 0.006037714891135693, "Regression_sin_phi": 0.005946646444499493, "Regression_cos_phi": 0.0065452782437205315, "Regression_energy": 0.09857796132564545, "Total": 14.6530179977417, "Total_unweighted": 2.255742550827563, "metrics/particle/class_1/efficiency": 0.9515522718429565, "metrics/particle/class_1/pid_accuracy": 0.9859387874603271, "metrics/particle/class_2/efficiency": 0.5327102541923523, "metrics/particle/class_2/pid_accuracy": 0.8322368264198303, "metrics/particle/class_3/efficiency": 0.8318806886672974, "metrics/particle/class_3/pid_accuracy": 0.9894062876701355, "metrics/particle/class_4/efficiency": 0.899350643157959, "metrics/particle/class_4/pid_accuracy": 0.5342960357666016, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7573964595794678, "metrics/particle/count/bias_mean": 0.12800000607967377, "metrics/particle/count/mae": 2.9600000381469727, "metrics/particle/count/prediction_mean": 95.94000244140625, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.025820177048444748, "metrics/particle/event/energy_response_mean": 1.001810908317566, "metrics/particle/event/met_abs_error": 4.291443824768066, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025561805814504623, "metrics/particle/event/scalar_pt_response_mean": 1.0038225650787354, "metrics/particle/event/vector_pt_closure": 0.028057632967829704, "metrics/particle/matched/delta_eta_abs_mean": 0.00862474087625742, "metrics/particle/matched/delta_phi_abs_mean": 0.00983230210840702, "metrics/particle/matched/delta_r_mean": 0.014612546190619469, "metrics/particle/matched/energy_relative_abs_mean": 0.0872591957449913, "metrics/particle/matched/pid_accuracy": 0.9727783203125, "metrics/particle/matched/pt_relative_abs_mean": 0.08156929165124893, "metrics/particle/matching/duplicate_fraction": 0.026308109983801842, "metrics/particle/matching/efficiency": 0.869577944278717, "metrics/particle/matching/f1": 0.8689974546432495, "metrics/particle/matching/purity": 0.8684177398681641, "metrics/particle/matching/target_coverage_dr0p05": 0.8473260402679443, "metrics/particle/matching/target_coverage_dr0p10": 0.8986765742301941, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7651233673095703, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8976746201515198}, "step_train_time": 0.7624678611755371, "step_valid_time": 1.9805937960045412, "step_total_time": 1229.7360162734985, "elapsed_training_loop_seconds": 1229.7360162734985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033400189110255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06743426137609865, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1520, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9539776462853385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9546052631578947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9542913515291023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045394736842105265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9532894736842106, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9529759947385729, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9539776462853385, "step_test_time": 3.531652012025006, "step_plot_time": 2.718458298011683, "step_validation_cycle_time": 8.594655325054191} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..cc3556847e4d3dc3bcf2e4df06dad9d72af70fbd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 28.469750889679716, "train": {"Classification_binary": 1.93584144115448, "Classification": 0.013012044131755829, "Regression_pt": 0.09882770478725433, "Regression_eta": 0.0037510667461901903, "Regression_sin_phi": 0.0047990684397518635, "Regression_cos_phi": 0.0037428271025419235, "Regression_energy": 0.09466499090194702, "Total": 11.859274864196777, "Total_unweighted": 2.154639143263921}, "train_group": {}, "valid": {"Classification_binary": 2.0233850479125977, "Classification": 0.016379181295633316, "Regression_pt": 0.10010389983654022, "Regression_eta": 0.006129813380539417, "Regression_sin_phi": 0.006057298742234707, "Regression_cos_phi": 0.006516977213323116, "Regression_energy": 0.09843770414590836, "Total": 14.710380554199219, "Total_unweighted": 2.257009922526777, "metrics/particle/class_1/efficiency": 0.9505028128623962, "metrics/particle/class_1/pid_accuracy": 0.9862912893295288, "metrics/particle/class_2/efficiency": 0.5379672646522522, "metrics/particle/class_2/pid_accuracy": 0.8262757658958435, "metrics/particle/class_3/efficiency": 0.8264575004577637, "metrics/particle/class_3/pid_accuracy": 0.9891024231910706, "metrics/particle/class_4/efficiency": 0.9025974273681641, "metrics/particle/class_4/pid_accuracy": 0.564748227596283, "metrics/particle/class_5/efficiency": 0.9767441749572754, "metrics/particle/class_5/pid_accuracy": 0.761904776096344, "metrics/particle/count/bias_mean": -0.14000000059604645, "metrics/particle/count/mae": 3.1080000400543213, "metrics/particle/count/prediction_mean": 95.6719970703125, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027335016056895256, "metrics/particle/event/energy_response_mean": 0.9995883107185364, "metrics/particle/event/met_abs_error": 4.205173969268799, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026532260701060295, "metrics/particle/event/scalar_pt_response_mean": 1.001941442489624, "metrics/particle/event/vector_pt_closure": 0.02768886461853981, "metrics/particle/matched/delta_eta_abs_mean": 0.008852191269397736, "metrics/particle/matched/delta_phi_abs_mean": 0.010112559422850609, "metrics/particle/matched/delta_r_mean": 0.015028689056634903, "metrics/particle/matched/energy_relative_abs_mean": 0.08735382556915283, "metrics/particle/matched/pid_accuracy": 0.9728935956954956, "metrics/particle/matched/pt_relative_abs_mean": 0.08145067095756531, "metrics/particle/matching/duplicate_fraction": 0.026381803676486015, "metrics/particle/matching/efficiency": 0.867114782333374, "metrics/particle/matching/f1": 0.8677487373352051, "metrics/particle/matching/purity": 0.8683836460113525, "metrics/particle/matching/target_coverage_dr0p05": 0.8424832224845886, "metrics/particle/matching/target_coverage_dr0p10": 0.8953366875648499, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7642049193382263, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8959211707115173}, "step_train_time": 0.7665913105010986, "step_valid_time": 1.9753617500537075, "step_total_time": 1312.688134431839, "elapsed_training_loop_seconds": 1312.688134431839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020039608717908, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06558776957687062, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1450, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.955892034233048, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9552631578947368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044107965766951945, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533201840894149, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9545753785385122, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539473684210527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9546351084812623, "step_test_time": 3.1259793079807423, "step_plot_time": 2.731288621958811, "step_validation_cycle_time": 8.196884608012624} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..b15f941be57318a1b7fa37c7d998a84dadccc21b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 30.249110320284696, "train": {"Classification_binary": 1.8066744804382324, "Classification": 0.014056033454835415, "Regression_pt": 0.06530408561229706, "Regression_eta": 0.011616010218858719, "Regression_sin_phi": 0.004949386231601238, "Regression_cos_phi": 0.005169966258108616, "Regression_energy": 0.06449780613183975, "Total": 13.67622184753418, "Total_unweighted": 1.9722677683457732}, "train_group": {}, "valid": {"Classification_binary": 2.0318915843963623, "Classification": 0.016445517539978027, "Regression_pt": 0.10107037425041199, "Regression_eta": 0.006020831875503063, "Regression_sin_phi": 0.006031878758221865, "Regression_cos_phi": 0.006507457233965397, "Regression_energy": 0.099346823990345, "Total": 14.718125343322754, "Total_unweighted": 2.2673144680447876, "metrics/particle/class_1/efficiency": 0.9512898921966553, "metrics/particle/class_1/pid_accuracy": 0.9873138666152954, "metrics/particle/class_2/efficiency": 0.5350467562675476, "metrics/particle/class_2/pid_accuracy": 0.8198689818382263, "metrics/particle/class_3/efficiency": 0.8311059474945068, "metrics/particle/class_3/pid_accuracy": 0.9896294474601746, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5345454812049866, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7602339386940002, "metrics/particle/count/bias_mean": 0.05999999865889549, "metrics/particle/count/mae": 3.0280001163482666, "metrics/particle/count/prediction_mean": 95.87200164794922, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027285588905215263, "metrics/particle/event/energy_response_mean": 1.0013830661773682, "metrics/particle/event/met_abs_error": 4.388556480407715, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026992039754986763, "metrics/particle/event/scalar_pt_response_mean": 1.0026379823684692, "metrics/particle/event/vector_pt_closure": 0.029408039525151253, "metrics/particle/matched/delta_eta_abs_mean": 0.00866651814430952, "metrics/particle/matched/delta_phi_abs_mean": 0.009847180917859077, "metrics/particle/matched/delta_r_mean": 0.014654234051704407, "metrics/particle/matched/energy_relative_abs_mean": 0.08731820434331894, "metrics/particle/matched/pid_accuracy": 0.9730573296546936, "metrics/particle/matched/pt_relative_abs_mean": 0.08152362704277039, "metrics/particle/matching/duplicate_fraction": 0.027620160952210426, "metrics/particle/matching/efficiency": 0.8692857027053833, "metrics/particle/matching/f1": 0.8690136075019836, "metrics/particle/matching/purity": 0.8687416315078735, "metrics/particle/matching/target_coverage_dr0p05": 0.8487036824226379, "metrics/particle/matching/target_coverage_dr0p10": 0.8984260559082031, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7644971609115601, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8972571492195129}, "step_train_time": 0.8162603378295898, "step_valid_time": 1.9820288519840688, "step_total_time": 1395.082427740097, "elapsed_training_loop_seconds": 1395.082427740097, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001824314671831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06726632981836567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9559500328731098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528178243774574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9543813587134887, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047182175622542594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495412844036697, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9510994420741713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9559500328731098, "step_test_time": 3.171194753027521, "step_plot_time": 2.72958521702094, "step_validation_cycle_time": 8.238264983985573} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..be9138cb7ba0b2a7cd1b3a9b150c85c3bbb9e56f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 32.02846975088968, "train": {"Classification_binary": 1.9263544082641602, "Classification": 0.015010274946689606, "Regression_pt": 0.0792183056473732, "Regression_eta": 0.001953620696440339, "Regression_sin_phi": 0.0014742928324267268, "Regression_cos_phi": 0.0025682200212031603, "Regression_energy": 0.07816925644874573, "Total": 9.187591552734375, "Total_unweighted": 2.104748378857039}, "train_group": {}, "valid": {"Classification_binary": 2.034935712814331, "Classification": 0.01672067493200302, "Regression_pt": 0.10082901269197464, "Regression_eta": 0.006142929196357727, "Regression_sin_phi": 0.006106604356318712, "Regression_cos_phi": 0.006671617738902569, "Regression_energy": 0.09905582666397095, "Total": 14.871827125549316, "Total_unweighted": 2.2704623783938587, "metrics/particle/class_1/efficiency": 0.9491910934448242, "metrics/particle/class_1/pid_accuracy": 0.9866408705711365, "metrics/particle/class_2/efficiency": 0.548481285572052, "metrics/particle/class_2/pid_accuracy": 0.8264110684394836, "metrics/particle/class_3/efficiency": 0.8315901756286621, "metrics/particle/class_3/pid_accuracy": 0.988354504108429, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.5289855003356934, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7810651063919067, "metrics/particle/count/bias_mean": 0.14000000059604645, "metrics/particle/count/mae": 3.1480000019073486, "metrics/particle/count/prediction_mean": 95.9520034790039, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026741638779640198, "metrics/particle/event/energy_response_mean": 1.0027810335159302, "metrics/particle/event/met_abs_error": 4.507007598876953, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026452012360095978, "metrics/particle/event/scalar_pt_response_mean": 1.0039139986038208, "metrics/particle/event/vector_pt_closure": 0.030428823083639145, "metrics/particle/matched/delta_eta_abs_mean": 0.008805862627923489, "metrics/particle/matched/delta_phi_abs_mean": 0.009861001744866371, "metrics/particle/matched/delta_r_mean": 0.01477176696062088, "metrics/particle/matched/energy_relative_abs_mean": 0.08735623210668564, "metrics/particle/matched/pid_accuracy": 0.9723889827728271, "metrics/particle/matched/pt_relative_abs_mean": 0.08152404427528381, "metrics/particle/matching/duplicate_fraction": 0.02722194418311119, "metrics/particle/matching/efficiency": 0.8694109320640564, "metrics/particle/matching/f1": 0.8687762022018433, "metrics/particle/matching/purity": 0.8681424260139465, "metrics/particle/matching/target_coverage_dr0p05": 0.8488289713859558, "metrics/particle/matching/target_coverage_dr0p10": 0.8982173204421997, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7648728489875793, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8963804244995117}, "step_train_time": 0.9286050796508789, "step_valid_time": 1.9912382820039056, "step_total_time": 1477.9811613559723, "elapsed_training_loop_seconds": 1477.9811613559723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013913421160598, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0667622880491946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585798816568047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560655737704918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573210768220617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043934426229508196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572649572649573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9547540983606557, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9560078791858174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585798816568047, "step_test_time": 3.4026319780386984, "step_plot_time": 2.756952790019568, "step_validation_cycle_time": 8.507779710984323} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..f1e414515beb7b575b027fa01aa87d7a9b90b423 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 33.80782918149466, "train": {"Classification_binary": 1.9515222311019897, "Classification": 0.008511105552315712, "Regression_pt": 0.0725824385881424, "Regression_eta": 0.0028626692947000265, "Regression_sin_phi": 0.0029656891711056232, "Regression_cos_phi": 0.003070067847147584, "Regression_energy": 0.06976347416639328, "Total": 9.057945251464844, "Total_unweighted": 2.1112776757217944}, "train_group": {}, "valid": {"Classification_binary": 2.0402719974517822, "Classification": 0.016883576288819313, "Regression_pt": 0.10216356068849564, "Regression_eta": 0.0064208609983325005, "Regression_sin_phi": 0.006139154080301523, "Regression_cos_phi": 0.006588951218873262, "Regression_energy": 0.10047435760498047, "Total": 15.026046752929688, "Total_unweighted": 2.278942458331585, "metrics/particle/class_1/efficiency": 0.9482291340827942, "metrics/particle/class_1/pid_accuracy": 0.9872729182243347, "metrics/particle/class_2/efficiency": 0.5408878326416016, "metrics/particle/class_2/pid_accuracy": 0.823974072933197, "metrics/particle/class_3/efficiency": 0.8313964605331421, "metrics/particle/class_3/pid_accuracy": 0.9892836213111877, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5309090614318848, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7764706015586853, "metrics/particle/count/bias_mean": 0.18799999356269836, "metrics/particle/count/mae": 3.0840001106262207, "metrics/particle/count/prediction_mean": 96.0, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026538120582699776, "metrics/particle/event/energy_response_mean": 1.005446434020996, "metrics/particle/event/met_abs_error": 4.614465236663818, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026540154591202736, "metrics/particle/event/scalar_pt_response_mean": 1.0063183307647705, "metrics/particle/event/vector_pt_closure": 0.029919905588030815, "metrics/particle/matched/delta_eta_abs_mean": 0.00896504893898964, "metrics/particle/matched/delta_phi_abs_mean": 0.009986210614442825, "metrics/particle/matched/delta_r_mean": 0.014975403435528278, "metrics/particle/matched/energy_relative_abs_mean": 0.08724677562713623, "metrics/particle/matched/pid_accuracy": 0.9730756282806396, "metrics/particle/matched/pt_relative_abs_mean": 0.08166195452213287, "metrics/particle/matching/duplicate_fraction": 0.0267083328217268, "metrics/particle/matching/efficiency": 0.8683254718780518, "metrics/particle/matching/f1": 0.867474377155304, "metrics/particle/matching/purity": 0.8666250109672546, "metrics/particle/matching/target_coverage_dr0p05": 0.8456143140792847, "metrics/particle/matching/target_coverage_dr0p10": 0.8982173204421997, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7635369300842285, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8950444459915161}, "step_train_time": 0.803372859954834, "step_valid_time": 1.9833840149804018, "step_total_time": 1560.7831499576569, "elapsed_training_loop_seconds": 1560.7831499576569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035727873080422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06824559687348386, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9516023544800524, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9540983606557377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04839764551994768, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9496402877697842, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9521311475409836, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566074950690335, "step_test_time": 3.355336108012125, "step_plot_time": 2.7416374419699423, "step_validation_cycle_time": 8.442476109950803} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..9a1989813a574b1c084ba2b986c6ccba1a46301b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 3.5587188612099645, "train": {"Classification_binary": 2.1262338161468506, "Classification": 0.019302429631352425, "Regression_pt": 0.10266096144914627, "Regression_eta": 0.004023104906082153, "Regression_sin_phi": 0.004344508983194828, "Regression_cos_phi": 0.004912390373647213, "Regression_energy": 0.10308438539505005, "Total": 13.474874496459961, "Total_unweighted": 2.3645615968853235}, "train_group": {}, "valid": {"Classification_binary": 1.977612018585205, "Classification": 0.015612361952662468, "Regression_pt": 0.09781112521886826, "Regression_eta": 0.00584951089695096, "Regression_sin_phi": 0.005801704712212086, "Regression_cos_phi": 0.006395520642399788, "Regression_energy": 0.09590884298086166, "Total": 14.240044593811035, "Total_unweighted": 2.2049910849891603, "metrics/particle/class_1/efficiency": 0.9464800953865051, "metrics/particle/class_1/pid_accuracy": 0.9873417615890503, "metrics/particle/class_2/efficiency": 0.5455607771873474, "metrics/particle/class_2/pid_accuracy": 0.8276231288909912, "metrics/particle/class_3/efficiency": 0.8311059474945068, "metrics/particle/class_3/pid_accuracy": 0.9877650737762451, "metrics/particle/class_4/efficiency": 0.8863636255264282, "metrics/particle/class_4/pid_accuracy": 0.5311355590820312, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7633135914802551, "metrics/particle/count/bias_mean": 0.052000001072883606, "metrics/particle/count/mae": 2.947999954223633, "metrics/particle/count/prediction_mean": 95.86399841308594, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026335861533880234, "metrics/particle/event/energy_response_mean": 1.002468466758728, "metrics/particle/event/met_abs_error": 4.359381675720215, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026324838399887085, "metrics/particle/event/scalar_pt_response_mean": 1.0049951076507568, "metrics/particle/event/vector_pt_closure": 0.02820991910994053, "metrics/particle/matched/delta_eta_abs_mean": 0.00856534019112587, "metrics/particle/matched/delta_phi_abs_mean": 0.009702053852379322, "metrics/particle/matched/delta_r_mean": 0.014447659254074097, "metrics/particle/matched/energy_relative_abs_mean": 0.08627540618181229, "metrics/particle/matched/pid_accuracy": 0.9725229740142822, "metrics/particle/matched/pt_relative_abs_mean": 0.0814579650759697, "metrics/particle/matching/duplicate_fraction": 0.026078611612319946, "metrics/particle/matching/efficiency": 0.8675739765167236, "metrics/particle/matching/f1": 0.8673386573791504, "metrics/particle/matching/purity": 0.8671033978462219, "metrics/particle/matching/target_coverage_dr0p05": 0.8474094867706299, "metrics/particle/matching/target_coverage_dr0p10": 0.8977998495101929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7637456655502319, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8967561721801758}, "step_train_time": 0.8099968433380127, "step_valid_time": 1.9942677189828828, "step_total_time": 159.6226806640625, "elapsed_training_loop_seconds": 159.6226806640625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0041760404845443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0630193598336255, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585798816568047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560655737704918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573210768220617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043934426229508196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9559500328731098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534426229508197, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546946815495733, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585798816568047, "step_test_time": 3.185134469997138, "step_plot_time": 2.6749357850058004, "step_validation_cycle_time": 8.195245834998786} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..fdd7a5ea6e22cbbfe5bd7620ae4781b1c8a820c8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 1.9339693784713745, "Classification": 0.01260842103511095, "Regression_pt": 0.08106020838022232, "Regression_eta": 0.002522020135074854, "Regression_sin_phi": 0.0033063674345612526, "Regression_cos_phi": 0.002708925399929285, "Regression_energy": 0.08069158345460892, "Total": 9.856433868408203, "Total_unweighted": 2.116866904310882}, "train_group": {}, "valid": {"Classification_binary": 2.0386176109313965, "Classification": 0.01676991954445839, "Regression_pt": 0.10197947919368744, "Regression_eta": 0.006277518812566996, "Regression_sin_phi": 0.006220100913196802, "Regression_cos_phi": 0.006600932218134403, "Regression_energy": 0.10013999044895172, "Total": 14.985882759094238, "Total_unweighted": 2.2766055520623922, "metrics/particle/class_1/efficiency": 0.9507651925086975, "metrics/particle/class_1/pid_accuracy": 0.9860191345214844, "metrics/particle/class_2/efficiency": 0.5420560836791992, "metrics/particle/class_2/pid_accuracy": 0.8243534564971924, "metrics/particle/class_3/efficiency": 0.8274258971214294, "metrics/particle/class_3/pid_accuracy": 0.9888811111450195, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5490909218788147, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7836257219314575, "metrics/particle/count/bias_mean": -0.052000001072883606, "metrics/particle/count/mae": 3.124000072479248, "metrics/particle/count/prediction_mean": 95.76000213623047, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027351966127753258, "metrics/particle/event/energy_response_mean": 0.9998124241828918, "metrics/particle/event/met_abs_error": 4.631821155548096, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026988478377461433, "metrics/particle/event/scalar_pt_response_mean": 1.0014073848724365, "metrics/particle/event/vector_pt_closure": 0.029847024008631706, "metrics/particle/matched/delta_eta_abs_mean": 0.009041490033268929, "metrics/particle/matched/delta_phi_abs_mean": 0.010444791987538338, "metrics/particle/matched/delta_r_mean": 0.01540708914399147, "metrics/particle/matched/energy_relative_abs_mean": 0.0878143385052681, "metrics/particle/matched/pid_accuracy": 0.9725348949432373, "metrics/particle/matched/pt_relative_abs_mean": 0.0820809155702591, "metrics/particle/matching/duplicate_fraction": 0.026691729202866554, "metrics/particle/matching/efficiency": 0.8679497241973877, "metrics/particle/matching/f1": 0.8681853413581848, "metrics/particle/matching/purity": 0.8684210777282715, "metrics/particle/matching/target_coverage_dr0p05": 0.8403957486152649, "metrics/particle/matching/target_coverage_dr0p10": 0.8960464000701904, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7619922161102295, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8947939872741699}, "step_train_time": 0.8163633346557617, "step_valid_time": 2.0157459159963764, "step_total_time": 1643.0784425735474, "elapsed_training_loop_seconds": 1643.0784425735474, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011099312011804, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06672517057107175, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1447, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501639344262295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9514116874589625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049836065573770495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9513477975016437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9488524590163935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9500984898227183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9526627218934911, "step_test_time": 3.3303694460191764, "step_plot_time": 2.7309041019761935, "step_validation_cycle_time": 8.433707214018796} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_21000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..399563cf11f85e9a64de539ee5687ce561b6b845 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 37.36654804270463, "train": {"Classification_binary": 2.1073696613311768, "Classification": 0.010283245705068111, "Regression_pt": 0.07890211790800095, "Regression_eta": 0.0037038985174149275, "Regression_sin_phi": 0.003481872845441103, "Regression_cos_phi": 0.004787062294781208, "Regression_energy": 0.0790611132979393, "Total": 10.840385437011719, "Total_unweighted": 2.2875889718998224}, "train_group": {}, "valid": {"Classification_binary": 2.051382064819336, "Classification": 0.017110135406255722, "Regression_pt": 0.10285335034132004, "Regression_eta": 0.006433992646634579, "Regression_sin_phi": 0.006256099324673414, "Regression_cos_phi": 0.006739986594766378, "Regression_energy": 0.10109813511371613, "Total": 15.194194793701172, "Total_unweighted": 2.291873764246702, "metrics/particle/class_1/efficiency": 0.9493659734725952, "metrics/particle/class_1/pid_accuracy": 0.986735463142395, "metrics/particle/class_2/efficiency": 0.5280373692512512, "metrics/particle/class_2/pid_accuracy": 0.8296459913253784, "metrics/particle/class_3/efficiency": 0.8270385265350342, "metrics/particle/class_3/pid_accuracy": 0.9884074926376343, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.5510948896408081, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7352941036224365, "metrics/particle/count/bias_mean": -0.30799999833106995, "metrics/particle/count/mae": 3.1480000019073486, "metrics/particle/count/prediction_mean": 95.50399780273438, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02671019546687603, "metrics/particle/event/energy_response_mean": 1.0038609504699707, "metrics/particle/event/met_abs_error": 4.504092693328857, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02608029730618, "metrics/particle/event/scalar_pt_response_mean": 1.0051358938217163, "metrics/particle/event/vector_pt_closure": 0.029836369678378105, "metrics/particle/matched/delta_eta_abs_mean": 0.008777263574302197, "metrics/particle/matched/delta_phi_abs_mean": 0.01018720306456089, "metrics/particle/matched/delta_r_mean": 0.014995305798947811, "metrics/particle/matched/energy_relative_abs_mean": 0.088208869099617, "metrics/particle/matched/pid_accuracy": 0.972763180732727, "metrics/particle/matched/pt_relative_abs_mean": 0.08236285299062729, "metrics/particle/matching/duplicate_fraction": 0.02684704214334488, "metrics/particle/matching/efficiency": 0.8660293221473694, "metrics/particle/matching/f1": 0.8674235343933105, "metrics/particle/matching/purity": 0.8688222765922546, "metrics/particle/matching/target_coverage_dr0p05": 0.8451968431472778, "metrics/particle/matching/target_coverage_dr0p10": 0.8961716890335083, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7598212957382202, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8929570317268372}, "step_train_time": 0.8002560138702393, "step_valid_time": 1.9801972480490804, "step_total_time": 1726.0932574272156, "elapsed_training_loop_seconds": 1726.0932574272156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026266023279766, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06609588727153481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9572649572649573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9522563767168084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9547540983606557, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04774362328319163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9559500328731098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9509483322432962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534426229508197, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9572649572649573, "step_test_time": 3.21323452697834, "step_plot_time": 2.742479125969112, "step_validation_cycle_time": 8.303831327008083} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_22000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..a05e9923e89df0c76e18f5aa4666bb71a1c7f5c5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 39.145907473309606, "train": {"Classification_binary": 1.73646879196167, "Classification": 0.013253160752356052, "Regression_pt": 0.07068135589361191, "Regression_eta": 0.0025413064286112785, "Regression_sin_phi": 0.0027060911525040865, "Regression_cos_phi": 0.0019267218885943294, "Regression_energy": 0.06886614114046097, "Total": 8.801285743713379, "Total_unweighted": 1.8964435692178085}, "train_group": {}, "valid": {"Classification_binary": 2.0402426719665527, "Classification": 0.017042430117726326, "Regression_pt": 0.10290797054767609, "Regression_eta": 0.00619480898603797, "Regression_sin_phi": 0.006318251136690378, "Regression_cos_phi": 0.006763826124370098, "Regression_energy": 0.10150434076786041, "Total": 15.137262344360352, "Total_unweighted": 2.280974299646914, "metrics/particle/class_1/efficiency": 0.9503279328346252, "metrics/particle/class_1/pid_accuracy": 0.9856446385383606, "metrics/particle/class_2/efficiency": 0.5362149477005005, "metrics/particle/class_2/pid_accuracy": 0.8224400877952576, "metrics/particle/class_3/efficiency": 0.8302343487739563, "metrics/particle/class_3/pid_accuracy": 0.9886854290962219, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.5328466892242432, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7647058963775635, "metrics/particle/count/bias_mean": 0.23600000143051147, "metrics/particle/count/mae": 3.0999999046325684, "metrics/particle/count/prediction_mean": 96.0479965209961, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027240484952926636, "metrics/particle/event/energy_response_mean": 0.9979134202003479, "metrics/particle/event/met_abs_error": 4.432464122772217, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026612035930156708, "metrics/particle/event/scalar_pt_response_mean": 1.0003443956375122, "metrics/particle/event/vector_pt_closure": 0.029789315536618233, "metrics/particle/matched/delta_eta_abs_mean": 0.008851226419210434, "metrics/particle/matched/delta_phi_abs_mean": 0.010205891914665699, "metrics/particle/matched/delta_r_mean": 0.015085565857589245, "metrics/particle/matched/energy_relative_abs_mean": 0.08682145923376083, "metrics/particle/matched/pid_accuracy": 0.9719257950782776, "metrics/particle/matched/pt_relative_abs_mean": 0.08141914755105972, "metrics/particle/matching/duplicate_fraction": 0.026570048183202744, "metrics/particle/matching/efficiency": 0.8684507012367249, "metrics/particle/matching/f1": 0.8673824667930603, "metrics/particle/matching/purity": 0.8663168549537659, "metrics/particle/matching/target_coverage_dr0p05": 0.8442783951759338, "metrics/particle/matching/target_coverage_dr0p10": 0.8966726660728455, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.765290379524231, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.897382378578186}, "step_train_time": 0.8447730541229248, "step_valid_time": 1.9807589370175265, "step_total_time": 1809.2377481460571, "elapsed_training_loop_seconds": 1809.2377481460571, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997659200877698, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06572717620175272, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552925706771861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954664914586071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549786395004929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04533508541392904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520367936925098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9523496549457772, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552925706771861, "step_test_time": 3.2870801509707235, "step_plot_time": 2.7393514559953474, "step_validation_cycle_time": 8.371963798010256} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_23000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..4fcc8da4a7c44aa8a3013b3e413d32090055ee9f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 40.92526690391459, "train": {"Classification_binary": 1.9136720895767212, "Classification": 0.009900144301354885, "Regression_pt": 0.08053309470415115, "Regression_eta": 0.003056178567931056, "Regression_sin_phi": 0.0019799601286649704, "Regression_cos_phi": 0.0024276766926050186, "Regression_energy": 0.08214739710092545, "Total": 9.095870018005371, "Total_unweighted": 2.0937165410723537}, "train_group": {}, "valid": {"Classification_binary": 2.0587122440338135, "Classification": 0.017273640260100365, "Regression_pt": 0.10299451649188995, "Regression_eta": 0.006340426858514547, "Regression_sin_phi": 0.006308123003691435, "Regression_cos_phi": 0.006692369002848864, "Regression_energy": 0.10144589841365814, "Total": 15.20425796508789, "Total_unweighted": 2.299767218064517, "metrics/particle/class_1/efficiency": 0.948928713798523, "metrics/particle/class_1/pid_accuracy": 0.9864528775215149, "metrics/particle/class_2/efficiency": 0.5332943797111511, "metrics/particle/class_2/pid_accuracy": 0.8170865178108215, "metrics/particle/class_3/efficiency": 0.8285880088806152, "metrics/particle/class_3/pid_accuracy": 0.9901823401451111, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5454545617103577, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7823529243469238, "metrics/particle/count/bias_mean": -0.19200000166893005, "metrics/particle/count/mae": 3.135999917984009, "metrics/particle/count/prediction_mean": 95.62000274658203, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02823934331536293, "metrics/particle/event/energy_response_mean": 0.9939044117927551, "metrics/particle/event/met_abs_error": 4.427706718444824, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027423007413744926, "metrics/particle/event/scalar_pt_response_mean": 0.9965704679489136, "metrics/particle/event/vector_pt_closure": 0.029761474579572678, "metrics/particle/matched/delta_eta_abs_mean": 0.008915924467146397, "metrics/particle/matched/delta_phi_abs_mean": 0.01010632049292326, "metrics/particle/matched/delta_r_mean": 0.015062615275382996, "metrics/particle/matched/energy_relative_abs_mean": 0.08645498007535934, "metrics/particle/matched/pid_accuracy": 0.9730315208435059, "metrics/particle/matched/pt_relative_abs_mean": 0.08097679167985916, "metrics/particle/matching/duplicate_fraction": 0.026521647348999977, "metrics/particle/matching/efficiency": 0.8669060468673706, "metrics/particle/matching/f1": 0.8677754998207092, "metrics/particle/matching/purity": 0.8686467409133911, "metrics/particle/matching/target_coverage_dr0p05": 0.8431929349899292, "metrics/particle/matching/target_coverage_dr0p10": 0.8950444459915161, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7637456655502319, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8945017457008362}, "step_train_time": 0.8051421642303467, "step_valid_time": 2.0004983220133, "step_total_time": 1892.292634010315, "elapsed_training_loop_seconds": 1892.292634010315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9984070258791639, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06559518144664656, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1447, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1445, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9513477975016437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9513477975016437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9513477975016437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04865220249835635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9513477975016437, "step_test_time": 3.396520017005969, "step_plot_time": 2.724302227026783, "step_validation_cycle_time": 8.494386617967393} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_24000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..8337cc1067599d8ea708e775f444066b5c685a02 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 42.704626334519574, "train": {"Classification_binary": 1.8872100114822388, "Classification": 0.013323777355253696, "Regression_pt": 0.0701809674501419, "Regression_eta": 0.002089644782245159, "Regression_sin_phi": 0.00419765617698431, "Regression_cos_phi": 0.002822058042511344, "Regression_energy": 0.06958906352519989, "Total": 9.676229476928711, "Total_unweighted": 2.049413178814575}, "train_group": {}, "valid": {"Classification_binary": 2.0583720207214355, "Classification": 0.01742844097316265, "Regression_pt": 0.10404089838266373, "Regression_eta": 0.006481644697487354, "Regression_sin_phi": 0.0063675460405647755, "Regression_cos_phi": 0.00692799873650074, "Regression_energy": 0.10227983444929123, "Total": 15.414297103881836, "Total_unweighted": 2.301898384001106, "metrics/particle/class_1/efficiency": 0.9478793144226074, "metrics/particle/class_1/pid_accuracy": 0.9865301251411438, "metrics/particle/class_2/efficiency": 0.5432242751121521, "metrics/particle/class_2/pid_accuracy": 0.823655903339386, "metrics/particle/class_3/efficiency": 0.82829749584198, "metrics/particle/class_3/pid_accuracy": 0.9888927936553955, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.54347825050354, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7633135914802551, "metrics/particle/count/bias_mean": 0.01600000075995922, "metrics/particle/count/mae": 3.1600000858306885, "metrics/particle/count/prediction_mean": 95.8280029296875, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027031833305954933, "metrics/particle/event/energy_response_mean": 1.0003156661987305, "metrics/particle/event/met_abs_error": 4.656400203704834, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02733171172440052, "metrics/particle/event/scalar_pt_response_mean": 1.0026750564575195, "metrics/particle/event/vector_pt_closure": 0.03053259663283825, "metrics/particle/matched/delta_eta_abs_mean": 0.008829031139612198, "metrics/particle/matched/delta_phi_abs_mean": 0.010150541551411152, "metrics/particle/matched/delta_r_mean": 0.01500098966062069, "metrics/particle/matched/energy_relative_abs_mean": 0.08709705621004105, "metrics/particle/matched/pid_accuracy": 0.9725044369697571, "metrics/particle/matched/pt_relative_abs_mean": 0.08170836418867111, "metrics/particle/matching/duplicate_fraction": 0.027382394298911095, "metrics/particle/matching/efficiency": 0.8669894933700562, "metrics/particle/matching/f1": 0.8669171333312988, "metrics/particle/matching/purity": 0.8668447732925415, "metrics/particle/matching/target_coverage_dr0p05": 0.8449046015739441, "metrics/particle/matching/target_coverage_dr0p10": 0.896463930606842, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7636204361915588, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8957959413528442}, "step_train_time": 0.7634029388427734, "step_valid_time": 2.01568488398334, "step_total_time": 1975.1883473396301, "elapsed_training_loop_seconds": 1975.1883473396301, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002258111128053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06795372769762353, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552925706771861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9502943100065402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527868852459016, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04970568999345978, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9526627218934911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.947678221059516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9501639344262295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552925706771861, "step_test_time": 3.3160384880029596, "step_plot_time": 2.7372410820098594, "step_validation_cycle_time": 8.445010099036153} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_25000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..d0d057f0326877f0c51e4723ba3871deabb5656c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 44.48398576512456, "train": {"Classification_binary": 2.018808126449585, "Classification": 0.011047843843698502, "Regression_pt": 0.07104852795600891, "Regression_eta": 0.002378132427111268, "Regression_sin_phi": 0.002190143335610628, "Regression_cos_phi": 0.0018619537586346269, "Regression_energy": 0.06918565183877945, "Total": 8.547097206115723, "Total_unweighted": 2.1765203796094283}, "train_group": {}, "valid": {"Classification_binary": 2.061408519744873, "Classification": 0.017528535798192024, "Regression_pt": 0.10504642128944397, "Regression_eta": 0.0066362046636641026, "Regression_sin_phi": 0.006498122587800026, "Regression_cos_phi": 0.00701306015253067, "Regression_energy": 0.10369938611984253, "Total": 15.607084274291992, "Total_unweighted": 2.3078302503563464, "metrics/particle/class_1/efficiency": 0.948753833770752, "metrics/particle/class_1/pid_accuracy": 0.984422504901886, "metrics/particle/class_2/efficiency": 0.5420560836791992, "metrics/particle/class_2/pid_accuracy": 0.8200430870056152, "metrics/particle/class_3/efficiency": 0.8271353840827942, "metrics/particle/class_3/pid_accuracy": 0.9891113638877869, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.5364963412284851, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7882353067398071, "metrics/particle/count/bias_mean": 0.164000004529953, "metrics/particle/count/mae": 3.252000093460083, "metrics/particle/count/prediction_mean": 95.97599792480469, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.027778910472989082, "metrics/particle/event/energy_response_mean": 1.0060526132583618, "metrics/particle/event/met_abs_error": 4.383078575134277, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02752370573580265, "metrics/particle/event/scalar_pt_response_mean": 1.0083388090133667, "metrics/particle/event/vector_pt_closure": 0.029193444177508354, "metrics/particle/matched/delta_eta_abs_mean": 0.009423354640603065, "metrics/particle/matched/delta_phi_abs_mean": 0.01062733680009842, "metrics/particle/matched/delta_r_mean": 0.015849033370614052, "metrics/particle/matched/energy_relative_abs_mean": 0.08781474083662033, "metrics/particle/matched/pid_accuracy": 0.9714863896369934, "metrics/particle/matched/pt_relative_abs_mean": 0.08228109776973724, "metrics/particle/matching/duplicate_fraction": 0.02788197062909603, "metrics/particle/matching/efficiency": 0.8667807579040527, "metrics/particle/matching/f1": 0.8660395741462708, "metrics/particle/matching/purity": 0.865299642086029, "metrics/particle/matching/target_coverage_dr0p05": 0.8373898863792419, "metrics/particle/matching/target_coverage_dr0p10": 0.8959211707115173, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7616165280342102, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8953366875648499}, "step_train_time": 0.8144190311431885, "step_valid_time": 1.989545240008738, "step_total_time": 2057.834217071533, "elapsed_training_loop_seconds": 2057.834217071533, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005122706469777, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06927309528046521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1520, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1447, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9513477975016437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9519736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516606379480435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04802631578947368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9487179487179487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9493421052631579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490299243669845, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9513477975016437, "step_test_time": 3.136773528007325, "step_plot_time": 2.717189689981751, "step_validation_cycle_time": 8.212406328995712} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_26000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e90aa120cb5059eec65d44c00aac7213dbd0de9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 46.263345195729535, "train": {"Classification_binary": 1.9154859781265259, "Classification": 0.008290082216262817, "Regression_pt": 0.056037649512290955, "Regression_eta": 0.0045099700801074505, "Regression_sin_phi": 0.003198555437847972, "Regression_cos_phi": 0.002672487171366811, "Regression_energy": 0.054427891969680786, "Total": 8.815049171447754, "Total_unweighted": 2.0446226145140827}, "train_group": {}, "valid": {"Classification_binary": 2.0718719959259033, "Classification": 0.017651699483394623, "Regression_pt": 0.1053992435336113, "Regression_eta": 0.0065737697295844555, "Regression_sin_phi": 0.0064858924597501755, "Regression_cos_phi": 0.006973767653107643, "Regression_energy": 0.10401241481304169, "Total": 15.608624458312988, "Total_unweighted": 2.318968783598393, "metrics/particle/class_1/efficiency": 0.949715793132782, "metrics/particle/class_1/pid_accuracy": 0.9843462109565735, "metrics/particle/class_2/efficiency": 0.5227803587913513, "metrics/particle/class_2/pid_accuracy": 0.8234636783599854, "metrics/particle/class_3/efficiency": 0.8234553337097168, "metrics/particle/class_3/pid_accuracy": 0.9892978668212891, "metrics/particle/class_4/efficiency": 0.8831169009208679, "metrics/particle/class_4/pid_accuracy": 0.5551470518112183, "metrics/particle/class_5/efficiency": 0.9767441749572754, "metrics/particle/class_5/pid_accuracy": 0.7916666865348816, "metrics/particle/count/bias_mean": -0.24799999594688416, "metrics/particle/count/mae": 3.2160000801086426, "metrics/particle/count/prediction_mean": 95.56400299072266, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02869943343102932, "metrics/particle/event/energy_response_mean": 0.9976363182067871, "metrics/particle/event/met_abs_error": 4.817987442016602, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027510158717632294, "metrics/particle/event/scalar_pt_response_mean": 0.9988596439361572, "metrics/particle/event/vector_pt_closure": 0.03145524114370346, "metrics/particle/matched/delta_eta_abs_mean": 0.00901282299309969, "metrics/particle/matched/delta_phi_abs_mean": 0.010172323323786259, "metrics/particle/matched/delta_r_mean": 0.01516747660934925, "metrics/particle/matched/energy_relative_abs_mean": 0.08773446828126907, "metrics/particle/matched/pid_accuracy": 0.9722195267677307, "metrics/particle/matched/pt_relative_abs_mean": 0.08218870311975479, "metrics/particle/matching/duplicate_fraction": 0.02632790617644787, "metrics/particle/matching/efficiency": 0.8641088604927063, "metrics/particle/matching/f1": 0.8652286529541016, "metrics/particle/matching/purity": 0.8663513660430908, "metrics/particle/matching/target_coverage_dr0p05": 0.8397278189659119, "metrics/particle/matching/target_coverage_dr0p10": 0.8939172625541687, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7574416399002075, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8931657671928406}, "step_train_time": 0.8059418201446533, "step_valid_time": 1.9957364319707267, "step_total_time": 2140.3998322486877, "elapsed_training_loop_seconds": 2140.3998322486877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004101450763554, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06864108626724685, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9539776462853385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514754098360656, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527248850952068, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04852459016393443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520052596975674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495081967213115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9507550886408405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9539776462853385, "step_test_time": 3.2940877299988642, "step_plot_time": 2.76027480897028, "step_validation_cycle_time": 8.433256612042896} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_27000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..8c6122903df178bfa91b1bcf063ba31d86d4a99d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 48.04270462633452, "train": {"Classification_binary": 2.1280579566955566, "Classification": 0.013543414883315563, "Regression_pt": 0.08972213417291641, "Regression_eta": 0.0029576101806014776, "Regression_sin_phi": 0.0025378877762705088, "Regression_cos_phi": 0.0037660927046090364, "Regression_energy": 0.08607715368270874, "Total": 10.70635986328125, "Total_unweighted": 2.3266622500959784}, "train_group": {}, "valid": {"Classification_binary": 2.0790300369262695, "Classification": 0.017820585519075394, "Regression_pt": 0.1060652807354927, "Regression_eta": 0.006694885902106762, "Regression_sin_phi": 0.006574897561222315, "Regression_cos_phi": 0.007113246712833643, "Regression_energy": 0.1046333983540535, "Total": 15.785358428955078, "Total_unweighted": 2.327932331711054, "metrics/particle/class_1/efficiency": 0.9505903124809265, "metrics/particle/class_1/pid_accuracy": 0.983440637588501, "metrics/particle/class_2/efficiency": 0.5344626307487488, "metrics/particle/class_2/pid_accuracy": 0.8174863457679749, "metrics/particle/class_3/efficiency": 0.8303312063217163, "metrics/particle/class_3/pid_accuracy": 0.9897364377975464, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.55474454164505, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7810651063919067, "metrics/particle/count/bias_mean": 0.2199999988079071, "metrics/particle/count/mae": 3.0920000076293945, "metrics/particle/count/prediction_mean": 96.03199768066406, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026595260947942734, "metrics/particle/event/energy_response_mean": 1.0019739866256714, "metrics/particle/event/met_abs_error": 4.526186943054199, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02636590227484703, "metrics/particle/event/scalar_pt_response_mean": 1.0032726526260376, "metrics/particle/event/vector_pt_closure": 0.03067116066813469, "metrics/particle/matched/delta_eta_abs_mean": 0.009147809818387032, "metrics/particle/matched/delta_phi_abs_mean": 0.010385631583631039, "metrics/particle/matched/delta_r_mean": 0.015439205802977085, "metrics/particle/matched/energy_relative_abs_mean": 0.08726735413074493, "metrics/particle/matched/pid_accuracy": 0.9714450240135193, "metrics/particle/matched/pt_relative_abs_mean": 0.08153644949197769, "metrics/particle/matching/duplicate_fraction": 0.026282906532287598, "metrics/particle/matching/efficiency": 0.8684507012367249, "metrics/particle/matching/f1": 0.8674548268318176, "metrics/particle/matching/purity": 0.8664611577987671, "metrics/particle/matching/target_coverage_dr0p05": 0.8424832224845886, "metrics/particle/matching/target_coverage_dr0p10": 0.8968396186828613, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7643718719482422, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8977163434028625}, "step_train_time": 0.8106598854064941, "step_valid_time": 1.976629976008553, "step_total_time": 2223.6378614902496, "elapsed_training_loop_seconds": 2223.6378614902496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001435589095511, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06637876761618333, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1520, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9460880999342538, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9467105263157894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9463992107859257, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053289473684210525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9441157133464826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447368421052632, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9444261756001315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9460880999342538, "step_test_time": 3.3738783490261994, "step_plot_time": 2.7253502369858325, "step_validation_cycle_time": 8.446304474025965} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_28000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..cb5126c809b55488201a09ec1f66584482186eb8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 49.8220640569395, "train": {"Classification_binary": 1.982114553451538, "Classification": 0.0102669857442379, "Regression_pt": 0.06381979584693909, "Regression_eta": 0.005082476418465376, "Regression_sin_phi": 0.0036395154893398285, "Regression_cos_phi": 0.00305728567764163, "Regression_energy": 0.06271810829639435, "Total": 9.949922561645508, "Total_unweighted": 2.1306987209245563}, "train_group": {}, "valid": {"Classification_binary": 2.086230993270874, "Classification": 0.018122702836990356, "Regression_pt": 0.1059107556939125, "Regression_eta": 0.006789529230445623, "Regression_sin_phi": 0.006487269885838032, "Regression_cos_phi": 0.0071770017966628075, "Regression_energy": 0.1045735776424408, "Total": 15.850190162658691, "Total_unweighted": 2.335291830357164, "metrics/particle/class_1/efficiency": 0.9478793144226074, "metrics/particle/class_1/pid_accuracy": 0.9856997728347778, "metrics/particle/class_2/efficiency": 0.53387850522995, "metrics/particle/class_2/pid_accuracy": 0.8140043616294861, "metrics/particle/class_3/efficiency": 0.8276196122169495, "metrics/particle/class_3/pid_accuracy": 0.9892347455024719, "metrics/particle/class_4/efficiency": 0.8798701167106628, "metrics/particle/class_4/pid_accuracy": 0.5461254715919495, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7941176295280457, "metrics/particle/count/bias_mean": -0.07599999755620956, "metrics/particle/count/mae": 3.252000093460083, "metrics/particle/count/prediction_mean": 95.73600006103516, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02610408142209053, "metrics/particle/event/energy_response_mean": 0.9993946552276611, "metrics/particle/event/met_abs_error": 4.4260334968566895, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02573426626622677, "metrics/particle/event/scalar_pt_response_mean": 1.000694751739502, "metrics/particle/event/vector_pt_closure": 0.029978683218359947, "metrics/particle/matched/delta_eta_abs_mean": 0.009171582758426666, "metrics/particle/matched/delta_phi_abs_mean": 0.010430500842630863, "metrics/particle/matched/delta_r_mean": 0.015477503649890423, "metrics/particle/matched/energy_relative_abs_mean": 0.08714084327220917, "metrics/particle/matched/pid_accuracy": 0.9722757935523987, "metrics/particle/matched/pt_relative_abs_mean": 0.08169971406459808, "metrics/particle/matching/duplicate_fraction": 0.026990892365574837, "metrics/particle/matching/efficiency": 0.8658623099327087, "metrics/particle/matching/f1": 0.8662058711051941, "metrics/particle/matching/purity": 0.8665496706962585, "metrics/particle/matching/target_coverage_dr0p05": 0.8398113250732422, "metrics/particle/matching/target_coverage_dr0p10": 0.8932075500488281, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7632446885108948, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8952114582061768}, "step_train_time": 0.7703409194946289, "step_valid_time": 1.9759507880080491, "step_total_time": 2306.685387611389, "elapsed_training_loop_seconds": 2306.685387611389, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025422965547763, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06649257176883927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1451, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552925706771861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954664914586071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549786395004929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04533508541392904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9539776462853385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533508541392904, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9536641472231351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552925706771861, "step_test_time": 3.2196031310013495, "step_plot_time": 2.7627266520285048, "step_validation_cycle_time": 8.33492205600487} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_29000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..89d111b1253714462192507c891b58b69c110c15 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 51.60142348754449, "train": {"Classification_binary": 2.0131723880767822, "Classification": 0.009126290678977966, "Regression_pt": 0.07919605821371078, "Regression_eta": 0.0027199394535273314, "Regression_sin_phi": 0.005138537846505642, "Regression_cos_phi": 0.003351653227582574, "Regression_energy": 0.0777038037776947, "Total": 10.328614234924316, "Total_unweighted": 2.1904086712747812}, "train_group": {}, "valid": {"Classification_binary": 2.087787389755249, "Classification": 0.01833212748169899, "Regression_pt": 0.10702542215585709, "Regression_eta": 0.0066556124947965145, "Regression_sin_phi": 0.006694894749671221, "Regression_cos_phi": 0.007182521745562553, "Regression_energy": 0.10553979873657227, "Total": 15.954607963562012, "Total_unweighted": 2.3392177671194077, "metrics/particle/class_1/efficiency": 0.9504153728485107, "metrics/particle/class_1/pid_accuracy": 0.9851858615875244, "metrics/particle/class_2/efficiency": 0.5262850522994995, "metrics/particle/class_2/pid_accuracy": 0.8235294222831726, "metrics/particle/class_3/efficiency": 0.8285880088806152, "metrics/particle/class_3/pid_accuracy": 0.9910004734992981, "metrics/particle/class_4/efficiency": 0.9025974273681641, "metrics/particle/class_4/pid_accuracy": 0.5215827226638794, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7692307829856873, "metrics/particle/count/bias_mean": 0.00800000037997961, "metrics/particle/count/mae": 3.1760001182556152, "metrics/particle/count/prediction_mean": 95.81999969482422, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02942185290157795, "metrics/particle/event/energy_response_mean": 0.9990805983543396, "metrics/particle/event/met_abs_error": 4.699336528778076, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028276430442929268, "metrics/particle/event/scalar_pt_response_mean": 0.9994533658027649, "metrics/particle/event/vector_pt_closure": 0.03107808530330658, "metrics/particle/matched/delta_eta_abs_mean": 0.00902330782264471, "metrics/particle/matched/delta_phi_abs_mean": 0.010452712886035442, "metrics/particle/matched/delta_r_mean": 0.015418034046888351, "metrics/particle/matched/energy_relative_abs_mean": 0.08722594380378723, "metrics/particle/matched/pid_accuracy": 0.9726073741912842, "metrics/particle/matched/pt_relative_abs_mean": 0.08121240139007568, "metrics/particle/matching/duplicate_fraction": 0.026758505031466484, "metrics/particle/matching/efficiency": 0.8671982884407043, "metrics/particle/matching/f1": 0.8671620488166809, "metrics/particle/matching/purity": 0.8671258687973022, "metrics/particle/matching/target_coverage_dr0p05": 0.8404375314712524, "metrics/particle/matching/target_coverage_dr0p10": 0.8936667442321777, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7642466425895691, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8962969183921814}, "step_train_time": 0.7971701622009277, "step_valid_time": 1.99212836305378, "step_total_time": 2388.601565361023, "elapsed_training_loop_seconds": 2388.601565361023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986179445019008, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0679104210278001, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1445, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9520052596975674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9513797634691196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516924088070983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04862023653088042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9500328731097962, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9494086727989488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497206703910615, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9520052596975674, "step_test_time": 3.2269366609980352, "step_plot_time": 2.7568597779609263, "step_validation_cycle_time": 8.348852397000883} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_3000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..d74d6691913afd1c965e030217e941014819f13e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 5.338078291814947, "train": {"Classification_binary": 1.8117990493774414, "Classification": 0.017058266326785088, "Regression_pt": 0.08341106027364731, "Regression_eta": 0.0060891625471413136, "Regression_sin_phi": 0.0030407339800149202, "Regression_cos_phi": 0.0038425566162914038, "Regression_energy": 0.08024492859840393, "Total": 11.822453498840332, "Total_unweighted": 2.0054857577197254}, "train_group": {}, "valid": {"Classification_binary": 1.9784719944000244, "Classification": 0.015796201303601265, "Regression_pt": 0.09797564893960953, "Regression_eta": 0.0057857041247189045, "Regression_sin_phi": 0.005773018579930067, "Regression_cos_phi": 0.006337691098451614, "Regression_energy": 0.09597846865653992, "Total": 14.21816635131836, "Total_unweighted": 2.2061187271028757, "metrics/particle/class_1/efficiency": 0.9483165740966797, "metrics/particle/class_1/pid_accuracy": 0.9874585270881653, "metrics/particle/class_2/efficiency": 0.548481285572052, "metrics/particle/class_2/pid_accuracy": 0.8328008651733398, "metrics/particle/class_3/efficiency": 0.8333333134651184, "metrics/particle/class_3/pid_accuracy": 0.9886112809181213, "metrics/particle/class_4/efficiency": 0.899350643157959, "metrics/particle/class_4/pid_accuracy": 0.5270758271217346, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7660818696022034, "metrics/particle/count/bias_mean": 0.23999999463558197, "metrics/particle/count/mae": 2.944000005722046, "metrics/particle/count/prediction_mean": 96.052001953125, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02656129002571106, "metrics/particle/event/energy_response_mean": 1.001948595046997, "metrics/particle/event/met_abs_error": 4.277777194976807, "metrics/particle/event/scalar_pt_relative_abs_error": 0.026167841628193855, "metrics/particle/event/scalar_pt_response_mean": 1.0039228200912476, "metrics/particle/event/vector_pt_closure": 0.027737343683838844, "metrics/particle/matched/delta_eta_abs_mean": 0.008493103086948395, "metrics/particle/matched/delta_phi_abs_mean": 0.009771494194865227, "metrics/particle/matched/delta_r_mean": 0.014469409361481667, "metrics/particle/matched/energy_relative_abs_mean": 0.08615090698003769, "metrics/particle/matched/pid_accuracy": 0.9730274677276611, "metrics/particle/matched/pt_relative_abs_mean": 0.08087988942861557, "metrics/particle/matching/duplicate_fraction": 0.02690209448337555, "metrics/particle/matching/efficiency": 0.8698701858520508, "metrics/particle/matching/f1": 0.8687820434570312, "metrics/particle/matching/purity": 0.8676966428756714, "metrics/particle/matching/target_coverage_dr0p05": 0.8487036824226379, "metrics/particle/matching/target_coverage_dr0p10": 0.8989270925521851, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7676282525062561, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8980503678321838}, "step_train_time": 0.7477505207061768, "step_valid_time": 1.9773993749986403, "step_total_time": 241.48201704025269, "elapsed_training_loop_seconds": 241.48201704025269, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017687150041477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.062247907890170984, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585798816568047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9617414248021108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9601580507079355, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03825857519788918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552925706771861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9584432717678101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9568653276259467, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585798816568047, "step_test_time": 3.455246586003341, "step_plot_time": 2.698263580969069, "step_validation_cycle_time": 8.492218987026718} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..b3a093d85870ee34a4ce0baccd09da0ed9199e42 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 1.8312230110168457, "Classification": 0.011132920160889626, "Regression_pt": 0.08767446130514145, "Regression_eta": 0.0022065534722059965, "Regression_sin_phi": 0.006418293341994286, "Regression_cos_phi": 0.0019312531221657991, "Regression_energy": 0.08642838150262833, "Total": 10.539937019348145, "Total_unweighted": 2.027014873921871}, "train_group": {}, "valid": {"Classification_binary": 2.0896098613739014, "Classification": 0.018494414165616035, "Regression_pt": 0.10813813656568527, "Regression_eta": 0.006750724744051695, "Regression_sin_phi": 0.00668430095538497, "Regression_cos_phi": 0.007356441579759121, "Regression_energy": 0.10660559684038162, "Total": 16.11174964904785, "Total_unweighted": 2.34363947622478, "metrics/particle/class_1/efficiency": 0.9490161538124084, "metrics/particle/class_1/pid_accuracy": 0.9849797487258911, "metrics/particle/class_2/efficiency": 0.5391355156898499, "metrics/particle/class_2/pid_accuracy": 0.8223185539245605, "metrics/particle/class_3/efficiency": 0.8311059474945068, "metrics/particle/class_3/pid_accuracy": 0.988930344581604, "metrics/particle/class_4/efficiency": 0.899350643157959, "metrics/particle/class_4/pid_accuracy": 0.5270758271217346, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7869822382926941, "metrics/particle/count/bias_mean": 0.2199999988079071, "metrics/particle/count/mae": 3.2760000228881836, "metrics/particle/count/prediction_mean": 96.03199768066406, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.029086383059620857, "metrics/particle/event/energy_response_mean": 0.9956589937210083, "metrics/particle/event/met_abs_error": 4.620977878570557, "metrics/particle/event/scalar_pt_relative_abs_error": 0.028485281392931938, "metrics/particle/event/scalar_pt_response_mean": 0.9967179298400879, "metrics/particle/event/vector_pt_closure": 0.030627984553575516, "metrics/particle/matched/delta_eta_abs_mean": 0.009314753115177155, "metrics/particle/matched/delta_phi_abs_mean": 0.010524958372116089, "metrics/particle/matched/delta_r_mean": 0.015705808997154236, "metrics/particle/matched/energy_relative_abs_mean": 0.08736136555671692, "metrics/particle/matched/pid_accuracy": 0.9716867804527283, "metrics/particle/matched/pt_relative_abs_mean": 0.08150632679462433, "metrics/particle/matching/duplicate_fraction": 0.02649116888642311, "metrics/particle/matching/efficiency": 0.8684924840927124, "metrics/particle/matching/f1": 0.8674964904785156, "metrics/particle/matching/purity": 0.8665028214454651, "metrics/particle/matching/target_coverage_dr0p05": 0.8369306325912476, "metrics/particle/matching/target_coverage_dr0p10": 0.8955036997795105, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7660000920295715, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8972988724708557}, "step_train_time": 0.8457205295562744, "step_valid_time": 1.9839152470231056, "step_total_time": 2471.502147912979, "elapsed_training_loop_seconds": 2471.502147912979, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9981745434926638, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06871831066151401, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9460880999342538, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9423706614276359, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442257217847769, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05762933857236411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434582511505588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9397511460379829, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9416010498687664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9460880999342538, "step_test_time": 3.475521174026653, "step_plot_time": 2.7561529710073955, "step_validation_cycle_time": 8.599188855965622} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_31000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..bd0e6160a557c8326f7a5e88a34db8271ef72ad0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 55.16014234875445, "train": {"Classification_binary": 1.8785133361816406, "Classification": 0.00961786787956953, "Regression_pt": 0.0626814067363739, "Regression_eta": 0.0053906068205833435, "Regression_sin_phi": 0.004786550533026457, "Regression_cos_phi": 0.0025955860037356615, "Regression_energy": 0.062323905527591705, "Total": 10.072266578674316, "Total_unweighted": 2.0259092596825212}, "train_group": {}, "valid": {"Classification_binary": 2.0997228622436523, "Classification": 0.018567530438303947, "Regression_pt": 0.10749496519565582, "Regression_eta": 0.007100233808159828, "Regression_sin_phi": 0.006835315842181444, "Regression_cos_phi": 0.007436090148985386, "Regression_energy": 0.10630319267511368, "Total": 16.307655334472656, "Total_unweighted": 2.3534601903520525, "metrics/particle/class_1/efficiency": 0.948578953742981, "metrics/particle/class_1/pid_accuracy": 0.9840508699417114, "metrics/particle/class_2/efficiency": 0.5321261882781982, "metrics/particle/class_2/pid_accuracy": 0.8166849613189697, "metrics/particle/class_3/efficiency": 0.8261669278144836, "metrics/particle/class_3/pid_accuracy": 0.989450216293335, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.55474454164505, "metrics/particle/class_5/efficiency": 0.9825581312179565, "metrics/particle/class_5/pid_accuracy": 0.7988165616989136, "metrics/particle/count/bias_mean": -0.23600000143051147, "metrics/particle/count/mae": 3.124000072479248, "metrics/particle/count/prediction_mean": 95.57599639892578, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02774428389966488, "metrics/particle/event/energy_response_mean": 0.9991687536239624, "metrics/particle/event/met_abs_error": 4.485842704772949, "metrics/particle/event/scalar_pt_relative_abs_error": 0.027005204930901527, "metrics/particle/event/scalar_pt_response_mean": 0.999346137046814, "metrics/particle/event/vector_pt_closure": 0.030429957434535027, "metrics/particle/matched/delta_eta_abs_mean": 0.009355115704238415, "metrics/particle/matched/delta_phi_abs_mean": 0.010323354974389076, "metrics/particle/matched/delta_r_mean": 0.015557290986180305, "metrics/particle/matched/energy_relative_abs_mean": 0.08676399290561676, "metrics/particle/matched/pid_accuracy": 0.9717345237731934, "metrics/particle/matched/pt_relative_abs_mean": 0.08135949820280075, "metrics/particle/matching/duplicate_fraction": 0.025822382420301437, "metrics/particle/matching/efficiency": 0.8655283451080322, "metrics/particle/matching/f1": 0.8665956258773804, "metrics/particle/matching/purity": 0.8676655292510986, "metrics/particle/matching/target_coverage_dr0p05": 0.8375986218452454, "metrics/particle/matching/target_coverage_dr0p10": 0.8931657671928406, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.762576699256897, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.894000768661499}, "step_train_time": 0.7990832328796387, "step_valid_time": 2.0163291460485198, "step_total_time": 2554.7167904376984, "elapsed_training_loop_seconds": 2554.7167904376984, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995068537555568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07082919503074869, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1524, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1446, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1440, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9506903353057199, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9488188976377953, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9497536945812808, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051181102362204724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467455621301775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9448818897637795, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9458128078817734, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9506903353057199, "step_test_time": 3.1533539179945365, "step_plot_time": 2.811575165949762, "step_validation_cycle_time": 8.361045333964285} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_4000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..4a0daf9665c478f8902eb46002ec8e2146c3dd0d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 7.117437722419929, "train": {"Classification_binary": 1.8005003929138184, "Classification": 0.015623477287590504, "Regression_pt": 0.09279049187898636, "Regression_eta": 0.004816219210624695, "Regression_sin_phi": 0.005891692359000444, "Regression_cos_phi": 0.00733390124514699, "Regression_energy": 0.08863499760627747, "Total": 13.851465225219727, "Total_unweighted": 2.015591172501445}, "train_group": {}, "valid": {"Classification_binary": 1.982412576675415, "Classification": 0.0157233327627182, "Regression_pt": 0.09811802208423615, "Regression_eta": 0.005787995643913746, "Regression_sin_phi": 0.005819654557853937, "Regression_cos_phi": 0.006370133254677057, "Regression_energy": 0.09621617943048477, "Total": 14.249366760253906, "Total_unweighted": 2.210447894409299, "metrics/particle/class_1/efficiency": 0.949715793132782, "metrics/particle/class_1/pid_accuracy": 0.9861878156661987, "metrics/particle/class_2/efficiency": 0.5473130941390991, "metrics/particle/class_2/pid_accuracy": 0.8249733448028564, "metrics/particle/class_3/efficiency": 0.8345922827720642, "metrics/particle/class_3/pid_accuracy": 0.9881643056869507, "metrics/particle/class_4/efficiency": 0.9025974273681641, "metrics/particle/class_4/pid_accuracy": 0.5323740839958191, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7660818696022034, "metrics/particle/count/bias_mean": 0.41600000858306885, "metrics/particle/count/mae": 2.9119999408721924, "metrics/particle/count/prediction_mean": 96.22799682617188, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026723923161625862, "metrics/particle/event/energy_response_mean": 1.0023632049560547, "metrics/particle/event/met_abs_error": 4.316100597381592, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02606569230556488, "metrics/particle/event/scalar_pt_response_mean": 1.004165530204773, "metrics/particle/event/vector_pt_closure": 0.027677644044160843, "metrics/particle/matched/delta_eta_abs_mean": 0.008636544458568096, "metrics/particle/matched/delta_phi_abs_mean": 0.009621365927159786, "metrics/particle/matched/delta_r_mean": 0.014444584026932716, "metrics/particle/matched/energy_relative_abs_mean": 0.0868784636259079, "metrics/particle/matched/pid_accuracy": 0.9719133377075195, "metrics/particle/matched/pt_relative_abs_mean": 0.08143748342990875, "metrics/particle/matching/duplicate_fraction": 0.02676975540816784, "metrics/particle/matching/efficiency": 0.871039092540741, "metrics/particle/matching/f1": 0.8691522479057312, "metrics/particle/matching/purity": 0.8672735691070557, "metrics/particle/matching/target_coverage_dr0p05": 0.8477435111999512, "metrics/particle/matching/target_coverage_dr0p10": 0.9003047347068787, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7669603228569031, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8998037576675415}, "step_train_time": 0.8277993202209473, "step_valid_time": 1.9895459219696932, "step_total_time": 323.63238644599915, "elapsed_training_loop_seconds": 323.63238644599915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027859674705872, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06235816067988964, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1453, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1450, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552925706771861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540380827314511, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954664914586071, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04596191726854892, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533201840894149, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520682862770847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526938239159002, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552925706771861, "step_test_time": 3.2108931690454483, "step_plot_time": 2.6871574810356833, "step_validation_cycle_time": 8.235383615014143} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_5000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a36e4641bb2cb302ef65b6bda5ca4fd923e6c05 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 8.896797153024911, "train": {"Classification_binary": 1.8055975437164307, "Classification": 0.012265645898878574, "Regression_pt": 0.09689363092184067, "Regression_eta": 0.0023287117946892977, "Regression_sin_phi": 0.002145867794752121, "Regression_cos_phi": 0.006254648324102163, "Regression_energy": 0.09416913241147995, "Total": 11.081635475158691, "Total_unweighted": 2.0196551808621734}, "train_group": {}, "valid": {"Classification_binary": 1.9869787693023682, "Classification": 0.015859080478549004, "Regression_pt": 0.09798695892095566, "Regression_eta": 0.005816207733005285, "Regression_sin_phi": 0.005851146765053272, "Regression_cos_phi": 0.006442367564886808, "Regression_energy": 0.09608004242181778, "Total": 14.31276798248291, "Total_unweighted": 2.215014573186636, "metrics/particle/class_1/efficiency": 0.949715793132782, "metrics/particle/class_1/pid_accuracy": 0.9867403507232666, "metrics/particle/class_2/efficiency": 0.5473130941390991, "metrics/particle/class_2/pid_accuracy": 0.8281750082969666, "metrics/particle/class_3/efficiency": 0.8327522873878479, "metrics/particle/class_3/pid_accuracy": 0.9880218505859375, "metrics/particle/class_4/efficiency": 0.899350643157959, "metrics/particle/class_4/pid_accuracy": 0.5270758271217346, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7719298005104065, "metrics/particle/count/bias_mean": 0.30000001192092896, "metrics/particle/count/mae": 2.868000030517578, "metrics/particle/count/prediction_mean": 96.11199951171875, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026117367669939995, "metrics/particle/event/energy_response_mean": 1.0027529001235962, "metrics/particle/event/met_abs_error": 4.12846040725708, "metrics/particle/event/scalar_pt_relative_abs_error": 0.02497680112719536, "metrics/particle/event/scalar_pt_response_mean": 1.0032025575637817, "metrics/particle/event/vector_pt_closure": 0.027248285710811615, "metrics/particle/matched/delta_eta_abs_mean": 0.008395875804126263, "metrics/particle/matched/delta_phi_abs_mean": 0.00960281677544117, "metrics/particle/matched/delta_r_mean": 0.014243995770812035, "metrics/particle/matched/energy_relative_abs_mean": 0.08746635913848877, "metrics/particle/matched/pid_accuracy": 0.9722701907157898, "metrics/particle/matched/pt_relative_abs_mean": 0.08144815266132355, "metrics/particle/matching/duplicate_fraction": 0.02601131983101368, "metrics/particle/matching/efficiency": 0.8702041506767273, "metrics/particle/matching/f1": 0.8688439130783081, "metrics/particle/matching/purity": 0.867487907409668, "metrics/particle/matching/target_coverage_dr0p05": 0.8515008687973022, "metrics/particle/matching/target_coverage_dr0p10": 0.8988018035888672, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7655408382415771, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.9000959992408752}, "step_train_time": 0.7361748218536377, "step_valid_time": 1.965361226000823, "step_total_time": 405.77378034591675, "elapsed_training_loop_seconds": 405.77378034591675, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017102727576987, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06298149573537226, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1460, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9598948060486522, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9611586570111915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9605263157894737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.038841342988808425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572649572649573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585253456221198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9578947368421052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9598948060486522, "step_test_time": 3.4288560939603485, "step_plot_time": 2.6858072109753266, "step_validation_cycle_time": 8.41793586401036} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_6000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..fc09252be06c097eefdaac4f0567e32da10e40bb --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 10.676156583629894, "train": {"Classification_binary": 2.049962043762207, "Classification": 0.012293148785829544, "Regression_pt": 0.08314447849988937, "Regression_eta": 0.00520032923668623, "Regression_sin_phi": 0.005218155682086945, "Regression_cos_phi": 0.005347190424799919, "Regression_energy": 0.08106531202793121, "Total": 12.479015350341797, "Total_unweighted": 2.2422306584194303}, "train_group": {}, "valid": {"Classification_binary": 1.9916168451309204, "Classification": 0.015920490026474, "Regression_pt": 0.09841040521860123, "Regression_eta": 0.005799648817628622, "Regression_sin_phi": 0.005769191309809685, "Regression_cos_phi": 0.006336740683764219, "Regression_energy": 0.09648575633764267, "Total": 14.269634246826172, "Total_unweighted": 2.220339077524841, "metrics/particle/class_1/efficiency": 0.9499781131744385, "metrics/particle/class_1/pid_accuracy": 0.9864678382873535, "metrics/particle/class_2/efficiency": 0.542640209197998, "metrics/particle/class_2/pid_accuracy": 0.8180839419364929, "metrics/particle/class_3/efficiency": 0.8321712017059326, "metrics/particle/class_3/pid_accuracy": 0.9881298542022705, "metrics/particle/class_4/efficiency": 0.8961039185523987, "metrics/particle/class_4/pid_accuracy": 0.52173912525177, "metrics/particle/class_5/efficiency": 0.9941860437393188, "metrics/particle/class_5/pid_accuracy": 0.7836257219314575, "metrics/particle/count/bias_mean": 0.0560000017285347, "metrics/particle/count/mae": 2.936000108718872, "metrics/particle/count/prediction_mean": 95.86799621582031, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026006389409303665, "metrics/particle/event/energy_response_mean": 1.0018600225448608, "metrics/particle/event/met_abs_error": 4.2257819175720215, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025212476029992104, "metrics/particle/event/scalar_pt_response_mean": 1.0021580457687378, "metrics/particle/event/vector_pt_closure": 0.028010165318846703, "metrics/particle/matched/delta_eta_abs_mean": 0.008650376461446285, "metrics/particle/matched/delta_phi_abs_mean": 0.009591087698936462, "metrics/particle/matched/delta_r_mean": 0.014459889382123947, "metrics/particle/matched/energy_relative_abs_mean": 0.08762720227241516, "metrics/particle/matched/pid_accuracy": 0.971822202205658, "metrics/particle/matched/pt_relative_abs_mean": 0.0815494954586029, "metrics/particle/matching/duplicate_fraction": 0.025827178731560707, "metrics/particle/matching/efficiency": 0.8697031736373901, "metrics/particle/matching/f1": 0.8694490790367126, "metrics/particle/matching/purity": 0.8691951632499695, "metrics/particle/matching/target_coverage_dr0p05": 0.8485367298126221, "metrics/particle/matching/target_coverage_dr0p10": 0.8986765742301941, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7639544010162354, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.897382378578186}, "step_train_time": 0.7864751815795898, "step_valid_time": 2.010540973977186, "step_total_time": 488.1804528236389, "elapsed_training_loop_seconds": 488.1804528236389, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010733656806354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06398767760615631, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1518, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9565217391304348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9555774925962488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043478260869565216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9520052596975674, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9538866930171278, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9529450477130635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9546351084812623, "step_test_time": 3.308157832012512, "step_plot_time": 2.727561247011181, "step_validation_cycle_time": 8.394888676994015} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_7000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..654a5839c323f5a033e979a266493f8a3c8a4129 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 12.455516014234876, "train": {"Classification_binary": 1.9808732271194458, "Classification": 0.016515087336301804, "Regression_pt": 0.08142662048339844, "Regression_eta": 0.006305936723947525, "Regression_sin_phi": 0.004280770663172007, "Regression_cos_phi": 0.0032453674357384443, "Regression_energy": 0.08158320933580399, "Total": 12.20470905303955, "Total_unweighted": 2.174230219097808}, "train_group": {}, "valid": {"Classification_binary": 1.996256947517395, "Classification": 0.016063673421740532, "Regression_pt": 0.09836645424365997, "Regression_eta": 0.005795885808765888, "Regression_sin_phi": 0.0058737085200846195, "Regression_cos_phi": 0.006313193589448929, "Regression_energy": 0.09654016047716141, "Total": 14.320965766906738, "Total_unweighted": 2.2252100235782564, "metrics/particle/class_1/efficiency": 0.9512898921966553, "metrics/particle/class_1/pid_accuracy": 0.9864864945411682, "metrics/particle/class_2/efficiency": 0.5414719581604004, "metrics/particle/class_2/pid_accuracy": 0.8220064640045166, "metrics/particle/class_3/efficiency": 0.82820063829422, "metrics/particle/class_3/pid_accuracy": 0.9887745380401611, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5345454812049866, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7647058963775635, "metrics/particle/count/bias_mean": 0.11999999731779099, "metrics/particle/count/mae": 2.9600000381469727, "metrics/particle/count/prediction_mean": 95.93199920654297, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.024798233062028885, "metrics/particle/event/energy_response_mean": 1.0007989406585693, "metrics/particle/event/met_abs_error": 4.212702751159668, "metrics/particle/event/scalar_pt_relative_abs_error": 0.024559179320931435, "metrics/particle/event/scalar_pt_response_mean": 1.0010032653808594, "metrics/particle/event/vector_pt_closure": 0.027519576251506805, "metrics/particle/matched/delta_eta_abs_mean": 0.00844818726181984, "metrics/particle/matched/delta_phi_abs_mean": 0.009975623339414597, "metrics/particle/matched/delta_r_mean": 0.014590389095246792, "metrics/particle/matched/energy_relative_abs_mean": 0.08631531149148941, "metrics/particle/matched/pid_accuracy": 0.9723103642463684, "metrics/particle/matched/pt_relative_abs_mean": 0.08085693418979645, "metrics/particle/matching/duplicate_fraction": 0.026643872261047363, "metrics/particle/matching/efficiency": 0.8684507012367249, "metrics/particle/matching/f1": 0.8679072260856628, "metrics/particle/matching/purity": 0.8673644065856934, "metrics/particle/matching/target_coverage_dr0p05": 0.8462823033332825, "metrics/particle/matching/target_coverage_dr0p10": 0.8969231247901917, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7658748626708984, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.897382378578186}, "step_train_time": 0.814765214920044, "step_valid_time": 2.016542696976103, "step_total_time": 570.9591400623322, "elapsed_training_loop_seconds": 570.9591400623322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002973000576532, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.062467494024921466, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1520, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9566074950690335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9572368421052632, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569220651101611, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04276315789473684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9552631578947368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.954949029924367, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9566074950690335, "step_test_time": 3.4475965640158392, "step_plot_time": 2.7035367430071346, "step_validation_cycle_time": 8.532135314017069} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_8000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..92552850f67ad4eede6e6c64e2ff8b6f33a75c26 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 14.234875444839858, "train": {"Classification_binary": 1.9557898044586182, "Classification": 0.014800399541854858, "Regression_pt": 0.07724029570817947, "Regression_eta": 0.008792087435722351, "Regression_sin_phi": 0.004521245136857033, "Regression_cos_phi": 0.005828182213008404, "Regression_energy": 0.07823286950588226, "Total": 13.620100975036621, "Total_unweighted": 2.1452048840001225}, "train_group": {}, "valid": {"Classification_binary": 1.9922246932983398, "Classification": 0.01599995419383049, "Regression_pt": 0.09824149310588837, "Regression_eta": 0.0059235719963908195, "Regression_sin_phi": 0.005848572589457035, "Regression_cos_phi": 0.006371794268488884, "Regression_energy": 0.09645102173089981, "Total": 14.35753059387207, "Total_unweighted": 2.2210611011832952, "metrics/particle/class_1/efficiency": 0.9505028128623962, "metrics/particle/class_1/pid_accuracy": 0.9867513179779053, "metrics/particle/class_2/efficiency": 0.5449766516685486, "metrics/particle/class_2/pid_accuracy": 0.8242229223251343, "metrics/particle/class_3/efficiency": 0.8333333134651184, "metrics/particle/class_3/pid_accuracy": 0.9891923069953918, "metrics/particle/class_4/efficiency": 0.9025974273681641, "metrics/particle/class_4/pid_accuracy": 0.5323740839958191, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7882353067398071, "metrics/particle/count/bias_mean": 0.3240000009536743, "metrics/particle/count/mae": 2.9719998836517334, "metrics/particle/count/prediction_mean": 96.13600158691406, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.026276005432009697, "metrics/particle/event/energy_response_mean": 1.0015252828598022, "metrics/particle/event/met_abs_error": 4.275437355041504, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025690048933029175, "metrics/particle/event/scalar_pt_response_mean": 1.003649115562439, "metrics/particle/event/vector_pt_closure": 0.02864621765911579, "metrics/particle/matched/delta_eta_abs_mean": 0.008427340537309647, "metrics/particle/matched/delta_phi_abs_mean": 0.00962867122143507, "metrics/particle/matched/delta_r_mean": 0.014296679757535458, "metrics/particle/matched/energy_relative_abs_mean": 0.08621416240930557, "metrics/particle/matched/pid_accuracy": 0.972812294960022, "metrics/particle/matched/pt_relative_abs_mean": 0.08106482774019241, "metrics/particle/matching/duplicate_fraction": 0.02712823450565338, "metrics/particle/matching/efficiency": 0.8706634044647217, "metrics/particle/matching/f1": 0.8691937327384949, "metrics/particle/matching/purity": 0.8677290678024292, "metrics/particle/matching/target_coverage_dr0p05": 0.8489124774932861, "metrics/particle/matching/target_coverage_dr0p10": 0.8987182974815369, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7680457830429077, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8992610573768616}, "step_train_time": 0.7501316070556641, "step_valid_time": 2.0004802409675904, "step_total_time": 653.635589838028, "elapsed_training_loop_seconds": 653.635589838028, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013598784888365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06256914423343407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1518, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9572649572649573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9591567852437418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9582099374794341, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04084321475625823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9546351084812623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9565217391304348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9555774925962488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9572649572649573, "step_test_time": 3.1181958870147355, "step_plot_time": 2.7145379799767397, "step_validation_cycle_time": 8.1895809849957} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_9000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..17c2399e32beac0dc9eeb6f858904f9fbe7a39de --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 16.01423487544484, "train": {"Classification_binary": 1.9463059902191162, "Classification": 0.010514311492443085, "Regression_pt": 0.07625507563352585, "Regression_eta": 0.005037756636738777, "Regression_sin_phi": 0.002888305112719536, "Regression_cos_phi": 0.006515862885862589, "Regression_energy": 0.07479684799909592, "Total": 11.410672187805176, "Total_unweighted": 2.122314149979502}, "train_group": {}, "valid": {"Classification_binary": 1.9980508089065552, "Classification": 0.01601744070649147, "Regression_pt": 0.09853498637676239, "Regression_eta": 0.005844291765242815, "Regression_sin_phi": 0.00583578459918499, "Regression_cos_phi": 0.006348687689751387, "Regression_energy": 0.09658821672201157, "Total": 14.3359375, "Total_unweighted": 2.227220216766, "metrics/particle/class_1/efficiency": 0.9494534134864807, "metrics/particle/class_1/pid_accuracy": 0.9869208931922913, "metrics/particle/class_2/efficiency": 0.5496495366096497, "metrics/particle/class_2/pid_accuracy": 0.8225291967391968, "metrics/particle/class_3/efficiency": 0.8311059474945068, "metrics/particle/class_3/pid_accuracy": 0.989163339138031, "metrics/particle/class_4/efficiency": 0.8928571343421936, "metrics/particle/class_4/pid_accuracy": 0.5345454812049866, "metrics/particle/class_5/efficiency": 0.9883720874786377, "metrics/particle/class_5/pid_accuracy": 0.7882353067398071, "metrics/particle/count/bias_mean": -0.004000000189989805, "metrics/particle/count/mae": 2.940000057220459, "metrics/particle/count/prediction_mean": 95.80799865722656, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.02637924626469612, "metrics/particle/event/energy_response_mean": 1.0025767087936401, "metrics/particle/event/met_abs_error": 4.438505172729492, "metrics/particle/event/scalar_pt_relative_abs_error": 0.025874629616737366, "metrics/particle/event/scalar_pt_response_mean": 1.0024969577789307, "metrics/particle/event/vector_pt_closure": 0.02831062115728855, "metrics/particle/matched/delta_eta_abs_mean": 0.008503944613039494, "metrics/particle/matched/delta_phi_abs_mean": 0.009756348095834255, "metrics/particle/matched/delta_r_mean": 0.014477876015007496, "metrics/particle/matched/energy_relative_abs_mean": 0.08772505819797516, "metrics/particle/matched/pid_accuracy": 0.9728211164474487, "metrics/particle/matched/pt_relative_abs_mean": 0.08124516159296036, "metrics/particle/matching/duplicate_fraction": 0.02659485675394535, "metrics/particle/matching/efficiency": 0.8694109320640564, "metrics/particle/matching/f1": 0.8694290518760681, "metrics/particle/matching/purity": 0.8694472312927246, "metrics/particle/matching/target_coverage_dr0p05": 0.8486202359199524, "metrics/particle/matching/target_coverage_dr0p10": 0.8983843326568604, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7660000920295715, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8965056538581848}, "step_train_time": 0.757676362991333, "step_valid_time": 2.0000984499929473, "step_total_time": 735.2360436916351, "elapsed_training_loop_seconds": 735.2360436916351, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002477106319386, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06252675887284265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1459, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9618671926364234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9606040709126723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9612352168199737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03939592908732764, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9592373438527285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9579776756401839, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9586070959264126, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9618671926364234, "step_test_time": 3.3659695560345426, "step_plot_time": 2.7130822220351547, "step_validation_cycle_time": 8.431722063047346} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..bf075ac3b345ec0d2b11cdac8c86075b705f98b9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 30, "checkpoint_freq": 2000, "val_freq": 1000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", "relaxed_load": false, "load_weights_only": true, "reset_task_loss_weights": true, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bcfa70059bc6ff0768bda263afc9a794d610b6fc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b458fdfd744cd2889219f1fbcd94f4cf471cabb631d33a1942b96b342d6cfa3c +size 4498 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..935669db12463d2c341fe72af60191fa8a09f42f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..5148a992ee1fbe3c2ebc8d4dbf30f0ad583c6fcd Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c70e1d6c13e8137956cc6aeb49baeec6553d3d7f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..5c53a09168d24f2d15676856369fe21ca3a69aa6 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..035eb9cddb62d21b80cc4025f79f6d250437c305 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..90c312cf8445ce988975d29322afa647ed5c83be Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c40ee744f0aeffd1988bad612458610e3c092941 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..c0b84874ccd4626644e97ee797f5b54bae2a89a4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a96a35d0f87bba8811598fb9fe2a8ecd0a7ad2e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..d60d40700a406a94689b7471a870657b0977a4d5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a32791191014f78a1a63adc6520cb8a1fa99a8c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..38bed32c6746955b5702eda81aef210fa21ac5e4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/plots_step_31000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/train/events.out.tfevents.1794734625.workergpu204.2114575.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/train/events.out.tfevents.1794734625.workergpu204.2114575.0 new file mode 100644 index 0000000000000000000000000000000000000000..47c6c2ee1d9dd72813adfc0ca6326ed746a96486 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/train/events.out.tfevents.1794734625.workergpu204.2114575.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:250c64db8a19baeef7e59ad34384543eb876a033f18fc2f41c5ac984aed30ab9 +size 1140189 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/valid/events.out.tfevents.1794734625.workergpu204.2114575.1 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/valid/events.out.tfevents.1794734625.workergpu204.2114575.1 new file mode 100644 index 0000000000000000000000000000000000000000..6ac4050382560f66b3833a30ee252b3f3e299528 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/runs/valid/events.out.tfevents.1794734625.workergpu204.2114575.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71d3634433ed41f386208e14be5ed3326d9423a783aae1129492957d755f6a49 +size 15823312 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c9935bc3f19ab1f8297fb534399d3e2c7ca79c91 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/scenario-manifest.json @@ -0,0 +1,1280 @@ +{ + "scenario": { + "name": "pf_fm_finetuning", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all" + }, + "allowed_variant_differences": [ + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets", + "data_config" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_finetuning", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_30mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 1000, + "checkpoint_freq": 2000, + "patience": 30, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "load_weights_only": true, + "reset_task_loss_weights": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.feature_bag": true, + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.trainable": "all", + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 30, + "checkpoint_freq": 2000, + "val_freq": 1000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "relaxed_load": false, + "load_weights_only": true, + "reset_task_loss_weights": true, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--load", + "experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "1000", + "--checkpoint_freq", + "2000", + "--patience", + "30", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--load_weights_only", + "true", + "--reset_task_loss_weights", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.feature_bag", + "true", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.trainable", + "all", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5afe0416615fb301a3101aadbe920ea3b33b9e8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +load_weights_only: true +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 30 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: true +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7f6b6172239bec916d8d41d181dc5b79259cec94 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log @@ -0,0 +1,2497 @@ +rank0 [2026-11-15 04:23:43,728] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-15 04:23:43,729] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-15 04:23:43,955] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 04:23:43,955] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 04:23:43,955] INFO : Instantiating model +rank0 [2026-11-15 04:23:43,973] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 04:23:43,973] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 04:23:43,974] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 04:23:43,975] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 04:23:43,975] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 04:23:43,975] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 04:23:43,975] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,977] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 04:23:43,977] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,979] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 04:23:43,979] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,981] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 04:23:43,981] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,983] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 04:23:43,983] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,985] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 04:23:43,985] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,988] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 04:23:43,988] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 04:23:43,988] INFO : Initializing output DNNs +rank0 [2026-11-15 04:23:43,989] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,991] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,997] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 04:23:43,997] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 04:23:43,997] INFO : jagged_attention enabled=True +rank0 [2026-11-15 04:23:43,997] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 04:23:43,997] INFO : task_queries enabled=True +rank0 [2026-11-15 04:23:43,998] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 04:23:43,998] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 04:23:43,998] INFO : classification_query parameters: 256 +rank0 [2026-11-15 04:23:43,998] INFO : regression_query parameters: 256 +rank0 [2026-11-15 04:23:43,998] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 04:23:43,998] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 04:23:43,998] INFO : output_mode=elementwise +rank0 [2026-11-15 04:23:43,998] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 04:23:43,998] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 04:23:43,998] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 04:23:43,998] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 04:23:43,999] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 04:23:43,999] INFO : MLPF __init__ done +rank0 [2026-11-15 04:23:43,999] INFO : Instantiated model +rank0 [2026-11-15 04:23:44,000] INFO : Moving model to device rank=0 +rank0 [2026-11-15 04:23:44,006] INFO : Moved model to device rank=0 +rank0 [2026-11-15 04:23:44,006] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 04:23:44,080] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 04:23:44,080] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 04:23:44,081] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 04:23:44,139] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 04:23:44,139] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 04:23:44,140] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 04:23:44,140] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 04:23:44,140] INFO : Total parameters: 3910930 +rank0 [2026-11-15 04:23:44,142] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 04:23:44,143] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334 +rank0 [2026-11-15 04:23:44,143] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334 +rank0 [2026-11-15 04:23:44,143] INFO : Getting dataloaders +rank0 [2026-11-15 04:23:44,143] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:44,303] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,351] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,387] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,428] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,468] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,533] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,585] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,632] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,670] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,710] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,710] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 04:23:44,710] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 04:23:44,710] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 04:23:44,710] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:44,723] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,732] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,744] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,757] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,770] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,781] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,792] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,803] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,817] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,828] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,829] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 04:23:44,829] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 04:23:44,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:45,030] INFO : Got dataloaders +rank0 [2026-11-15 04:23:45,031] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 04:23:45,031] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 04:23:45,031] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 04:23:45,038] INFO : Creating train_iterator +rank0 [2026-11-15 04:23:45,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:45,069] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:45,277] INFO : Created train_iterator +rank0 [2026-11-15 04:23:53,626] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9723292076587677, 'Classification': 0.014986821804195643, 'Regression_pt': 0.09762214239686727, 'Regression_eta': 0.006117531423224136, 'Regression_sin_phi': 0.0055219064909033475, 'Regression_cos_phi': 0.005466182068921625, 'Regression_energy': 0.09622789811342955}, weights={'Classification_binary': 1.0, 'Classification': 131.60423278808594, 'Regression_pt': 20.203706741333008, 'Regression_eta': 322.40606689453125, 'Regression_sin_phi': 357.1826477050781, 'Regression_cos_phi': 360.82391357421875, 'Regression_energy': 20.49643898010254} +rank0 [2026-11-15 04:24:27,826] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 04:24:27,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:00,125] INFO : Step 1000/500000 rank0 | Train Loss: 14.1385 | Train Loss Unweighted: 2.3414 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.0702 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1199 | Regression_eta: 0.0035 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.1109 | Classification_binary: 2.0702 | Regression_cos_phi: 2.0246 | Regression_energy: 2.4574 | Regression_eta: 1.1386 | Regression_pt: 2.4412 | Regression_sin_phi: 1.8957 | LR: 2.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7677s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0277 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2640 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.9 | uss_MB: 1351.1 | child_rss_MB: 23087.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:25:00,126] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:00,126] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:00,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:00,251] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 04:25:00,370] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:00,426] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:00,464] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:00,503] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:00,536] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:00,575] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:00,613] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:00,649] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:00,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:00,756] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:00,794] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:00,832] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:00,866] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:00,903] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:00,942] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:00,978] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:00,981] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:00,983] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:00,984] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:00,986] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:00,987] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:00,989] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:00,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:00,992] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:00,994] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:00,995] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:00,997] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:00,998] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:01,000] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:01,001] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:01,003] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:02,014] INFO : VALIDATION | Step=1000/500000 | Train Loss=14.1385 | Valid Loss=14.1806 | Train Loss Unweighted=2.3414 | Valid Loss Unweighted=2.2025 | Stale=0 | ETA=640.2m +rank0 [2026-11-15 04:25:02,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:02,501] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:02,509] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:02,747] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:03,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:03,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:03,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:03,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:03,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:04,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:04,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:04,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:04,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:04,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:04,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:05,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:05,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:05,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:05,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:05,705] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:05,712] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:06,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:06,381] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:07,332] INFO : Plotted jets +rank0 [2026-11-15 04:25:08,454] INFO : Plotted jet ratio +rank0 [2026-11-15 04:25:17,892] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 04:25:17,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:59,163] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 04:25:59,163] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:22,613] INFO : Step 2000/500000 rank0 | Train Loss: 13.4749 | Train Loss Unweighted: 2.3646 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.1262 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1031 | Regression_eta: 0.0040 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.5403 | Classification_binary: 2.1262 | Regression_cos_phi: 1.7725 | Regression_energy: 2.1129 | Regression_eta: 1.2971 | Regression_pt: 2.0741 | Regression_sin_phi: 1.5518 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8093s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0258 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2777.4 | uss_MB: 1514.0 | child_rss_MB: 23090.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:26:22,615] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 04:26:22,665] INFO : Running validation on rank0 +rank0 [2026-11-15 04:26:22,665] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:26:22,665] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:26:22,770] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 04:26:22,850] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:26:22,926] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:26:22,973] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:26:23,018] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:26:23,063] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:26:23,109] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:26:23,154] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:26:23,225] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:26:23,302] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:26:23,378] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:26:23,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:26:23,471] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:26:23,516] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:26:23,560] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:26:23,605] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:26:23,681] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:26:23,686] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:26:23,687] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:26:23,689] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:26:23,691] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:26:23,692] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:26:23,694] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:26:23,695] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:26:23,698] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:26:23,698] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:26:23,700] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:26:23,701] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:26:23,703] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:26:23,705] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:26:23,706] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:26:23,708] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:26:24,662] INFO : VALIDATION | Step=2000/500000 | Train Loss=13.4749 | Valid Loss=14.2400 | Train Loss Unweighted=2.3646 | Valid Loss Unweighted=2.2050 | Stale=1 | ETA=662.4m +rank0 [2026-11-15 04:26:24,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:26:24,814] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:26:24,821] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:24,867] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:26:25,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:26:25,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:26:25,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:26:25,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:26:26,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:26:26,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:26:26,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:26:26,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:26:26,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:26:26,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:26:26,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:26:27,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:26:27,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:26:27,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:26:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:26:27,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:26:27,846] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:26:27,854] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:28,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:28,464] INFO : Plotted number of elements +rank0 [2026-11-15 04:26:29,801] INFO : Plotted jets +rank0 [2026-11-15 04:26:30,514] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:48,903] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 04:26:48,903] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:30,158] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 04:27:30,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:44,540] INFO : Step 3000/500000 rank0 | Train Loss: 11.8225 | Train Loss Unweighted: 2.0055 | Train Loss Components: Classification: 0.0171 | Classification_binary: 1.8118 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0802 | Regression_eta: 0.0061 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.2449 | Classification_binary: 1.8118 | Regression_cos_phi: 1.3865 | Regression_energy: 1.6447 | Regression_eta: 1.9632 | Regression_pt: 1.6852 | Regression_sin_phi: 1.0861 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7470s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2396 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2834.5 | uss_MB: 1571.3 | child_rss_MB: 23093.8 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:27:44,541] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:44,541] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:44,541] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:44,643] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 04:27:44,724] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:44,801] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:44,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:44,894] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:44,940] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:44,987] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:45,034] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:45,105] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:45,182] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:45,259] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:45,307] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:45,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:45,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:45,444] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:45,490] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:45,566] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:45,571] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:45,573] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:45,574] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:45,576] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:45,578] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:45,579] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:45,581] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:45,582] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:45,584] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:45,586] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:45,587] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:45,589] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:45,590] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:45,592] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:45,593] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:46,534] INFO : VALIDATION | Step=3000/500000 | Train Loss=11.8225 | Valid Loss=14.2182 | Train Loss Unweighted=2.0055 | Valid Loss Unweighted=2.2061 | Stale=2 | ETA=666.8m +rank0 [2026-11-15 04:27:46,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:46,983] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:47,002] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:47,059] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:47,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:47,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:47,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:47,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:48,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:48,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:48,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:48,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:48,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:48,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:49,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:49,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:49,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:49,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:49,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:49,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:49,987] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:27:49,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:50,374] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:50,601] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:51,972] INFO : Plotted jets +rank0 [2026-11-15 04:27:52,679] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:20,149] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 04:28:20,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:01,035] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:29:01,035] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:06,628] INFO : Step 4000/500000 rank0 | Train Loss: 13.8515 | Train Loss Unweighted: 2.0156 | Train Loss Components: Classification: 0.0156 | Classification_binary: 1.8005 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0886 | Regression_eta: 0.0048 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.0561 | Classification_binary: 1.8005 | Regression_cos_phi: 2.6462 | Regression_energy: 1.8167 | Regression_eta: 1.5528 | Regression_pt: 1.8747 | Regression_sin_phi: 2.1044 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8267s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2918 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2868.9 | uss_MB: 1605.8 | child_rss_MB: 23098.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:29:06,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:29:06,679] INFO : Running validation on rank0 +rank0 [2026-11-15 04:29:06,679] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:29:06,679] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:29:06,780] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:29:06,859] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:29:06,936] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:29:06,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:29:07,029] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:29:07,074] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:29:07,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:29:07,166] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:29:07,237] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:29:07,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:29:07,389] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:29:07,435] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:29:07,479] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:29:07,524] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:29:07,570] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:29:07,615] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:29:07,679] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:29:07,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:29:07,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:29:07,688] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:29:07,689] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:29:07,690] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:29:07,692] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:29:07,694] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:29:07,695] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:29:07,698] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:29:07,698] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:29:07,700] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:29:07,702] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:29:07,704] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:29:07,705] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:29:07,707] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:29:08,674] INFO : VALIDATION | Step=4000/500000 | Train Loss=13.8515 | Valid Loss=14.2494 | Train Loss Unweighted=2.0156 | Valid Loss Unweighted=2.2104 | Stale=3 | ETA=668.8m +rank0 [2026-11-15 04:29:08,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:29:08,806] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:29:08,814] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:08,860] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:29:09,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:29:09,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:29:09,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:29:09,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:29:10,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:29:10,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:29:10,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:29:10,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:29:10,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:29:10,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:29:11,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:29:11,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:29:11,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:29:11,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:29:11,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:29:11,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:29:11,883] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:29:11,890] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:12,274] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:12,503] INFO : Plotted number of elements +rank0 [2026-11-15 04:29:13,848] INFO : Plotted jets +rank0 [2026-11-15 04:29:14,563] INFO : Plotted jet ratio +rank0 [2026-11-15 04:29:51,200] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:29:51,200] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:28,843] INFO : Step 5000/500000 rank0 | Train Loss: 11.0816 | Train Loss Unweighted: 2.0197 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.8056 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0942 | Regression_eta: 0.0023 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6142 | Classification_binary: 1.8056 | Regression_cos_phi: 2.2568 | Regression_energy: 1.9301 | Regression_eta: 0.7508 | Regression_pt: 1.9576 | Regression_sin_phi: 0.7665 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2888.7 | uss_MB: 1625.7 | child_rss_MB: 23097.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:28,845] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:28,845] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:28,845] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:28,946] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:30:29,024] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:29,101] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:29,149] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:29,194] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:29,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:29,284] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:29,331] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:29,402] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:29,478] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:29,554] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:29,601] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:29,646] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:29,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:29,739] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:29,785] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:29,850] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:29,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:29,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:29,858] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:29,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:29,861] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:29,862] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:29,864] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:29,865] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:29,867] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:29,869] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:29,870] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:29,872] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:29,873] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:29,875] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:29,876] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:30,813] INFO : VALIDATION | Step=5000/500000 | Train Loss=11.0816 | Valid Loss=14.3128 | Train Loss Unweighted=2.0197 | Valid Loss Unweighted=2.2150 | Stale=4 | ETA=669.5m +rank0 [2026-11-15 04:30:30,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:31,199] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:31,206] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:31,264] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:31,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:31,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:32,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:32,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:32,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:32,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:32,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:32,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:33,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:33,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:33,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:33,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:33,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:33,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:34,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:34,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:34,241] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:34,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:34,628] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:34,855] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:35,777] INFO : Plotted jets +rank0 [2026-11-15 04:30:36,919] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:41,483] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:30:41,483] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:22,628] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:31:22,628] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:51,141] INFO : Step 6000/500000 rank0 | Train Loss: 12.4790 | Train Loss Unweighted: 2.2422 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.0500 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0811 | Regression_eta: 0.0052 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6178 | Classification_binary: 2.0500 | Regression_cos_phi: 1.9294 | Regression_energy: 1.6616 | Regression_eta: 1.6766 | Regression_pt: 1.6798 | Regression_sin_phi: 1.8638 | LR: 2.19e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7855s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2832.1 | uss_MB: 1569.1 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 04:31:51,143] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:31:51,206] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:51,206] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:51,206] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:51,309] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:31:51,392] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:51,470] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:51,519] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:51,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:51,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:51,656] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:51,702] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:51,773] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:51,851] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:51,927] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:51,973] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:52,019] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:52,063] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:52,108] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:52,154] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:52,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:52,234] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:52,236] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:52,237] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:52,239] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:52,241] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:52,242] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:52,244] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:52,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:52,247] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:52,248] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:52,250] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:52,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:52,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:52,254] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:52,256] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:53,220] INFO : VALIDATION | Step=6000/500000 | Train Loss=12.4790 | Valid Loss=14.2696 | Train Loss Unweighted=2.2422 | Valid Loss Unweighted=2.2203 | Stale=5 | ETA=669.9m +rank0 [2026-11-15 04:31:53,220] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:53,548] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:53,554] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:53,602] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:54,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:54,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:54,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:54,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:54,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:54,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:55,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:55,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:55,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:55,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:55,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:55,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:55,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:56,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:56,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:56,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:56,526] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:56,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:56,917] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:57,145] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:58,529] INFO : Plotted jets +rank0 [2026-11-15 04:31:59,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:13,009] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:32:13,009] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:54,391] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:32:54,391] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:13,978] INFO : Step 7000/500000 rank0 | Train Loss: 12.2047 | Train Loss Unweighted: 2.1742 | Train Loss Components: Classification: 0.0165 | Classification_binary: 1.9809 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0816 | Regression_eta: 0.0063 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.1735 | Classification_binary: 1.9809 | Regression_cos_phi: 1.1710 | Regression_energy: 1.6722 | Regression_eta: 2.0331 | Regression_pt: 1.6451 | Regression_sin_phi: 1.5290 | LR: 2.26e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8139s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2938 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2942.3 | uss_MB: 1679.3 | child_rss_MB: 23101.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:33:13,979] INFO : Running validation on rank0 +rank0 [2026-11-15 04:33:13,979] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:33:13,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:33:14,087] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:33:14,169] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:33:14,246] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:33:14,294] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:33:14,339] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:33:14,385] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:33:14,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,478] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,550] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:33:14,626] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:33:14,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:33:14,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:33:14,794] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:33:14,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:33:14,884] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:33:14,930] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,007] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:33:15,012] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:33:15,013] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:33:15,015] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:33:15,016] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:33:15,018] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:33:15,020] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:33:15,021] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:33:15,023] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:33:15,024] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:33:15,026] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:33:15,027] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:33:15,029] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:33:15,030] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:33:15,032] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:33:15,034] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:33:16,000] INFO : VALIDATION | Step=7000/500000 | Train Loss=12.2047 | Valid Loss=14.3210 | Train Loss Unweighted=2.1742 | Valid Loss Unweighted=2.2252 | Stale=6 | ETA=670.2m +rank0 [2026-11-15 04:33:16,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:33:16,381] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:33:16,385] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:16,451] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:16,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:17,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:17,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:17,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:17,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:17,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:18,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:18,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:18,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:18,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:18,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:18,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:19,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:19,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:19,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:19,446] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:33:19,452] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:19,835] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:20,065] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:21,439] INFO : Plotted jets +rank0 [2026-11-15 04:33:22,146] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:44,770] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:33:44,770] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:26,222] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:34:26,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:36,620] INFO : Step 8000/500000 rank0 | Train Loss: 13.6201 | Train Loss Unweighted: 2.1452 | Train Loss Components: Classification: 0.0148 | Classification_binary: 1.9558 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0782 | Regression_eta: 0.0088 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.9478 | Classification_binary: 1.9558 | Regression_cos_phi: 2.1029 | Regression_energy: 1.6035 | Regression_eta: 2.8346 | Regression_pt: 1.5605 | Regression_sin_phi: 1.6149 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7494s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2423 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.4 | uss_MB: 1698.4 | child_rss_MB: 23098.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:36,621] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:34:36,672] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:36,672] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:36,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:36,774] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:34:36,854] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:36,931] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:36,980] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:37,025] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:37,071] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:37,117] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:37,162] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:37,234] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:37,311] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:37,387] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:37,433] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:37,478] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:37,523] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:37,569] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:37,615] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:37,678] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:37,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:37,684] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:37,686] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:37,687] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:37,689] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:37,690] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:37,692] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:37,693] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:37,695] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:37,697] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:37,698] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:37,700] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:37,701] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:37,703] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:37,704] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:38,675] INFO : VALIDATION | Step=8000/500000 | Train Loss=13.6201 | Valid Loss=14.3575 | Train Loss Unweighted=2.1452 | Valid Loss Unweighted=2.2211 | Stale=7 | ETA=670.0m +rank0 [2026-11-15 04:34:38,675] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:38,798] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:38,804] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:38,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:39,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:39,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:39,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:39,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:39,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:40,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:40,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:40,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:40,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:40,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:40,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:41,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:41,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:41,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:41,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:41,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:41,791] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:41,798] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:42,174] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:42,406] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:43,776] INFO : Plotted jets +rank0 [2026-11-15 04:34:44,501] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:16,277] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:35:16,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:56,995] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:35:56,996] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:58,271] INFO : Step 9000/500000 rank0 | Train Loss: 11.4107 | Train Loss Unweighted: 2.1223 | Train Loss Components: Classification: 0.0105 | Classification_binary: 1.9463 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0748 | Regression_eta: 0.0050 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3837 | Classification_binary: 1.9463 | Regression_cos_phi: 2.3511 | Regression_energy: 1.5331 | Regression_eta: 1.6242 | Regression_pt: 1.5406 | Regression_sin_phi: 1.0317 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7573s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2907.2 | uss_MB: 1644.3 | child_rss_MB: 23101.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:58,272] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:58,273] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:58,273] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:58,374] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:35:58,456] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:58,532] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:58,581] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:58,627] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:58,672] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:58,718] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:58,764] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:58,835] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:58,916] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:58,992] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:59,039] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:59,084] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:59,129] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:59,175] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:59,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:59,297] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:59,302] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:59,305] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:59,306] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:59,307] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:59,310] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:59,310] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:59,312] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:59,315] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:59,315] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:59,317] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:59,320] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:59,320] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:59,322] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:59,324] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:59,325] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:36:00,276] INFO : VALIDATION | Step=9000/500000 | Train Loss=11.4107 | Valid Loss=14.3359 | Train Loss Unweighted=2.1223 | Valid Loss Unweighted=2.2272 | Stale=8 | ETA=668.5m +rank0 [2026-11-15 04:36:00,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:36:00,598] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:36:00,602] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:00,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:36:01,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:36:01,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:36:01,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:36:01,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:36:01,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:36:01,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:36:02,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:36:02,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:36:02,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:36:02,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:36:02,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:36:02,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:36:03,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:36:03,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:36:03,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:36:03,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:36:03,641] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:36:03,647] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:04,033] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:04,262] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:05,181] INFO : Plotted jets +rank0 [2026-11-15 04:36:06,347] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:47,227] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:36:47,227] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:20,744] INFO : Step 10000/500000 rank0 | Train Loss: 10.2994 | Train Loss Unweighted: 1.7760 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.6080 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0725 | Regression_eta: 0.0041 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7006 | Classification_binary: 1.6080 | Regression_cos_phi: 1.3247 | Regression_energy: 1.4856 | Regression_eta: 1.3133 | Regression_pt: 1.4298 | Regression_sin_phi: 1.4373 | LR: 2.52e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7324s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0812 | max_tensorboard_logging: 0.2185 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3021.4 | uss_MB: 1758.4 | child_rss_MB: 23102.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:37:20,745] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:37:20,793] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:20,793] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:20,793] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:20,898] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:37:20,979] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:21,057] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:21,106] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:21,151] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:21,197] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:21,244] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:21,291] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:21,364] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:21,441] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:21,518] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:21,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:21,609] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:21,653] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:21,699] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,744] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,808] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:21,812] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:21,814] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:21,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:21,817] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:21,819] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:21,820] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:21,822] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:21,823] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:21,825] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:21,826] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:21,828] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:21,829] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:21,831] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:21,833] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:21,834] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:22,793] INFO : VALIDATION | Step=10000/500000 | Train Loss=10.2994 | Valid Loss=14.3647 | Train Loss Unweighted=1.7760 | Valid Loss Unweighted=2.2301 | Stale=9 | ETA=667.8m +rank0 [2026-11-15 04:37:22,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:22,923] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:22,927] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:22,975] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:23,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:23,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:23,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:23,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:24,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:24,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:24,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:24,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:24,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:24,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:25,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:25,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:25,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:25,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:25,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:25,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:25,920] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:25,927] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,318] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,553] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:27,469] INFO : Plotted jets +rank0 [2026-11-15 04:37:28,646] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:37,437] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:37:37,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:18,468] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:38:18,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:42,562] INFO : Step 11000/500000 rank0 | Train Loss: 21.5883 | Train Loss Unweighted: 2.4287 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.1267 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1425 | Regression_eta: 0.0263 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6069 | Classification_binary: 2.1267 | Regression_cos_phi: 2.1452 | Regression_energy: 2.9199 | Regression_eta: 8.4939 | Regression_pt: 2.2074 | Regression_sin_phi: 2.0884 | LR: 2.63e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3044.4 | uss_MB: 1781.4 | child_rss_MB: 23103.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:38:42,564] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:42,564] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:42,564] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:42,670] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:38:42,750] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:42,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:42,877] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:42,923] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:42,968] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:43,013] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:43,059] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:43,132] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:43,209] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:43,284] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:43,331] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:43,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:43,422] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:43,467] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:43,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:43,575] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:43,579] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:43,581] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:43,584] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:43,584] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:43,586] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:43,587] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:43,589] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:43,590] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:43,592] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:43,593] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:43,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:43,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:43,598] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:43,600] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:43,604] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:44,567] INFO : VALIDATION | Step=11000/500000 | Train Loss=21.5883 | Valid Loss=14.4309 | Train Loss Unweighted=2.4287 | Valid Loss Unweighted=2.2411 | Stale=10 | ETA=666.5m +rank0 [2026-11-15 04:38:44,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:44,879] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:44,883] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:44,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:45,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:45,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:45,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:45,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:46,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:46,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:46,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:46,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:46,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:46,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:46,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:47,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:47,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:47,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:47,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:47,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:47,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:47,883] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,269] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,501] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:49,442] INFO : Plotted jets +rank0 [2026-11-15 04:38:50,618] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:08,195] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:39:08,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:49,169] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:39:49,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:04,133] INFO : Step 12000/500000 rank0 | Train Loss: 10.2180 | Train Loss Unweighted: 2.0239 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.8560 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0752 | Regression_eta: 0.0036 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4378 | Classification_binary: 1.8560 | Regression_cos_phi: 1.5490 | Regression_energy: 1.5411 | Regression_eta: 1.1692 | Regression_pt: 1.4203 | Regression_sin_phi: 1.2446 | LR: 2.75e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7487s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0825 | max_tensorboard_logging: 0.2207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3053.5 | uss_MB: 1790.5 | child_rss_MB: 23105.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:04,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:40:04,183] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:04,183] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:04,183] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:04,285] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:40:04,365] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:04,443] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:04,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:04,536] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:04,581] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:04,627] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:04,673] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:04,745] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:04,823] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:04,899] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:04,946] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:04,992] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:05,038] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:05,083] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:05,129] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:05,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:05,197] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:05,198] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:05,201] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:05,202] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:05,203] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:05,206] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:05,206] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:05,208] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:05,210] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:05,211] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:05,212] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:05,215] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:05,216] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:05,218] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:05,219] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:06,154] INFO : VALIDATION | Step=12000/500000 | Train Loss=10.2180 | Valid Loss=14.5756 | Train Loss Unweighted=2.0239 | Valid Loss Unweighted=2.2481 | Stale=11 | ETA=665.0m +rank0 [2026-11-15 04:40:06,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:06,277] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:06,282] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:06,329] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:06,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:06,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:07,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:07,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:07,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:07,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:07,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:07,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:08,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:08,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:08,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:08,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:08,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:08,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:09,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:09,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:09,281] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:40:09,288] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,664] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,893] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:11,291] INFO : Plotted jets +rank0 [2026-11-15 04:40:12,008] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:39,347] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:40:39,347] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:20,998] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:41:20,998] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:27,196] INFO : Step 13000/500000 rank0 | Train Loss: 12.4893 | Train Loss Unweighted: 2.0903 | Train Loss Components: Classification: 0.0144 | Classification_binary: 1.8362 | Regression_cos_phi: 0.0038 | Regression_energy: 0.1128 | Regression_eta: 0.0044 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.8928 | Classification_binary: 1.8362 | Regression_cos_phi: 1.3571 | Regression_energy: 2.3123 | Regression_eta: 1.4283 | Regression_pt: 2.3225 | Regression_sin_phi: 1.3401 | LR: 2.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8134s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2525 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3087.5 | uss_MB: 1824.5 | child_rss_MB: 23104.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:27,197] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:27,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:27,197] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:27,307] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:41:27,384] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:27,461] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:27,511] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:27,557] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:27,602] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:27,649] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:27,695] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:27,767] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:27,843] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:27,919] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:27,966] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:28,011] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:28,056] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:28,102] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:28,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:28,211] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:28,215] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:28,217] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:28,219] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:28,220] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:28,222] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:28,223] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:28,225] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:28,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:28,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:28,229] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:28,231] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:28,232] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:28,234] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:28,235] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:28,237] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:29,193] INFO : VALIDATION | Step=13000/500000 | Train Loss=12.4893 | Valid Loss=14.5489 | Train Loss Unweighted=2.0903 | Valid Loss Unweighted=2.2405 | Stale=12 | ETA=664.4m +rank0 [2026-11-15 04:41:29,193] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:29,544] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:29,548] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:29,595] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:30,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:30,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:30,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:30,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:30,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:30,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:31,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:31,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:31,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:31,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:31,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:31,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:31,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:32,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:32,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:32,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:32,539] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:32,545] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:32,931] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:33,158] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:34,104] INFO : Plotted jets +rank0 [2026-11-15 04:41:35,284] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:11,767] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:42:11,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:50,283] INFO : Step 14000/500000 rank0 | Train Loss: 11.2762 | Train Loss Unweighted: 2.2799 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.0703 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0915 | Regression_eta: 0.0024 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.0224 | Classification_binary: 2.0703 | Regression_cos_phi: 1.5166 | Regression_energy: 1.8751 | Regression_eta: 0.7874 | Regression_pt: 1.8767 | Regression_sin_phi: 1.1277 | LR: 3.02e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0807 | max_tensorboard_logging: 0.2415 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3114.6 | uss_MB: 1851.6 | child_rss_MB: 23105.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:42:50,285] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:42:50,334] INFO : Running validation on rank0 +rank0 [2026-11-15 04:42:50,334] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:42:50,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:42:50,437] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:42:50,520] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:50,597] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:50,647] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:50,694] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:50,741] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:50,786] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:50,833] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:50,906] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:50,984] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:51,060] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:51,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:51,152] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:51,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:51,244] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:51,288] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:51,354] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:51,358] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:51,360] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:51,362] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:51,363] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:51,365] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:51,366] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:51,368] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:51,369] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:51,371] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:51,372] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:51,374] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:51,375] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:51,377] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:51,378] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:51,380] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:52,343] INFO : VALIDATION | Step=14000/500000 | Train Loss=11.2762 | Valid Loss=14.6595 | Train Loss Unweighted=2.2799 | Valid Loss Unweighted=2.2518 | Stale=13 | ETA=663.8m +rank0 [2026-11-15 04:42:52,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:52,473] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:52,478] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:52,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:52,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:53,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:53,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:53,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:53,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:53,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:53,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:54,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:54,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:54,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:54,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:54,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:54,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:55,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:55,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:55,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:55,448] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:42:55,454] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:55,845] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:56,080] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:57,006] INFO : Plotted jets +rank0 [2026-11-15 04:42:58,179] INFO : Plotted jet ratio +rank0 [2026-11-15 04:43:02,235] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:43:02,235] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:43,673] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:43:43,673] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:44:12,791] INFO : Step 15000/500000 rank0 | Train Loss: 11.8908 | Train Loss Unweighted: 1.9384 | Train Loss Components: Classification: 0.0125 | Classification_binary: 1.7427 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0836 | Regression_eta: 0.0036 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6463 | Classification_binary: 1.7427 | Regression_cos_phi: 1.9994 | Regression_energy: 1.7132 | Regression_eta: 1.1734 | Regression_pt: 1.7186 | Regression_sin_phi: 1.8971 | LR: 3.17e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7614s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2439 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3143.6 | uss_MB: 1880.6 | child_rss_MB: 23108.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:44:12,792] INFO : Running validation on rank0 +rank0 [2026-11-15 04:44:12,792] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:44:12,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:44:12,892] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:44:12,972] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:44:13,049] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:44:13,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:44:13,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:44:13,191] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:44:13,238] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:44:13,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:44:13,356] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:44:13,433] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:44:13,510] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:44:13,557] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:44:13,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:44:13,648] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:44:13,694] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:44:13,740] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:44:13,817] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:44:13,822] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:44:13,823] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:44:13,825] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:44:13,827] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:44:13,828] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:44:13,830] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:44:13,831] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:44:13,833] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:44:13,834] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:44:13,836] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:44:13,837] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:44:13,839] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:44:13,842] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:44:13,842] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:44:13,844] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:14,775] INFO : VALIDATION | Step=15000/500000 | Train Loss=11.8908 | Valid Loss=14.6530 | Train Loss Unweighted=1.9384 | Valid Loss Unweighted=2.2557 | Stale=14 | ETA=662.7m +rank0 [2026-11-15 04:44:14,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:15,168] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:15,172] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:15,232] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:15,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:15,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:16,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:16,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:16,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:16,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:16,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:16,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:17,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:17,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:17,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:17,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:17,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:17,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:18,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:18,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:18,305] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:44:18,312] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:18,695] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:18,924] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:19,844] INFO : Plotted jets +rank0 [2026-11-15 04:44:21,018] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:34,074] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:44:34,074] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:15,509] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:45:15,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:35,696] INFO : Step 16000/500000 rank0 | Train Loss: 11.8593 | Train Loss Unweighted: 2.1546 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9358 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0947 | Regression_eta: 0.0038 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7124 | Classification_binary: 1.9358 | Regression_cos_phi: 1.3505 | Regression_energy: 1.9403 | Regression_eta: 1.2094 | Regression_pt: 1.9967 | Regression_sin_phi: 1.7141 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3163.9 | uss_MB: 1900.9 | child_rss_MB: 23108.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:45:35,698] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:45:35,749] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:35,749] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:35,749] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:35,850] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:45:35,930] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:36,008] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:36,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:36,102] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:36,147] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:36,193] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:36,239] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:36,311] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:36,387] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:36,465] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:36,511] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:36,557] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:36,602] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:36,648] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:36,694] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:36,758] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:36,762] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:36,765] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:36,766] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:36,769] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:36,770] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:36,772] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:36,774] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:36,774] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:36,777] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:36,778] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:36,779] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:36,782] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:36,783] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:36,784] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:36,786] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:37,727] INFO : VALIDATION | Step=16000/500000 | Train Loss=11.8593 | Valid Loss=14.7104 | Train Loss Unweighted=2.1546 | Valid Loss Unweighted=2.2570 | Stale=15 | ETA=661.8m +rank0 [2026-11-15 04:45:37,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:37,833] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:37,837] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:37,884] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:38,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:38,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:38,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:38,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:38,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:39,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:39,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:39,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:39,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:39,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:40,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:40,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:40,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:40,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:40,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:40,852] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:45:40,870] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:41,255] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:41,484] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:42,844] INFO : Plotted jets +rank0 [2026-11-15 04:45:43,576] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:06,049] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:46:06,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:47,105] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:46:47,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:58,135] INFO : Step 17000/500000 rank0 | Train Loss: 13.6762 | Train Loss Unweighted: 1.9723 | Train Loss Components: Classification: 0.0141 | Classification_binary: 1.8067 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0645 | Regression_eta: 0.0116 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.8498 | Classification_binary: 1.8067 | Regression_cos_phi: 1.8654 | Regression_energy: 1.3220 | Regression_eta: 3.7451 | Regression_pt: 1.3194 | Regression_sin_phi: 1.7678 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8151s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0842 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3108.1 | uss_MB: 1845.1 | child_rss_MB: 23107.6 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:46:58,137] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:58,137] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:58,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:58,239] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:46:58,322] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:58,400] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:58,451] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:58,498] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:58,543] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:58,589] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:58,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:58,708] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:58,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:58,863] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:58,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:58,958] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:59,004] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:59,054] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:59,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:59,164] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:59,168] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:59,170] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:59,172] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:59,173] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:59,175] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:59,176] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:59,178] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:59,179] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:59,181] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:59,183] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:59,184] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:59,186] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:59,188] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:59,189] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:59,192] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:00,122] INFO : VALIDATION | Step=17000/500000 | Train Loss=13.6762 | Valid Loss=14.7181 | Train Loss Unweighted=1.9723 | Valid Loss Unweighted=2.2673 | Stale=16 | ETA=660.6m +rank0 [2026-11-15 04:47:00,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:00,234] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:47:00,241] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:00,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:00,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:00,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:01,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:01,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:01,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:01,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:01,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:47:01,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:47:02,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:47:02,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:47:02,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:47:02,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:47:02,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:47:02,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:47:03,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:47:03,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:47:03,291] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:47:03,298] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:03,682] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:03,910] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:04,825] INFO : Plotted jets +rank0 [2026-11-15 04:47:06,014] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:37,451] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:47:37,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:18,936] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:48:18,936] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:20,976] INFO : Step 18000/500000 rank0 | Train Loss: 9.1876 | Train Loss Unweighted: 2.1047 | Train Loss Components: Classification: 0.0150 | Classification_binary: 1.9264 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0782 | Regression_eta: 0.0020 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.9754 | Classification_binary: 1.9264 | Regression_cos_phi: 0.9267 | Regression_energy: 1.6022 | Regression_eta: 0.6299 | Regression_pt: 1.6005 | Regression_sin_phi: 0.5266 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.9282s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0025 | max_backward: 0.0273 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.3601 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3216.5 | uss_MB: 1953.5 | child_rss_MB: 23108.3 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:48:20,977] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:48:21,026] INFO : Running validation on rank0 +rank0 [2026-11-15 04:48:21,026] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:48:21,026] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:48:21,128] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:48:21,208] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:48:21,286] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:48:21,335] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:48:21,380] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:48:21,425] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:48:21,471] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:48:21,517] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:48:21,591] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:48:21,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:48:21,748] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:48:21,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:48:21,841] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:48:21,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:48:21,933] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:48:21,981] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:48:22,057] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:48:22,062] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:48:22,064] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:48:22,065] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:48:22,067] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:48:22,068] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:48:22,070] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:48:22,072] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:48:22,073] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:48:22,075] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:48:22,076] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:48:22,078] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:48:22,079] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:48:22,081] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:48:22,082] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:48:22,084] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:48:23,020] INFO : VALIDATION | Step=18000/500000 | Train Loss=9.1876 | Valid Loss=14.8718 | Train Loss Unweighted=2.1047 | Valid Loss Unweighted=2.2705 | Stale=17 | ETA=659.6m +rank0 [2026-11-15 04:48:23,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:48:23,347] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:48:23,353] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:23,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:48:23,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:48:24,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:48:24,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:48:24,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:48:24,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:48:24,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:48:24,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:25,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:25,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:25,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:25,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:25,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:25,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:26,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:26,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:26,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:26,421] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:48:26,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:26,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:27,041] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:27,971] INFO : Plotted jets +rank0 [2026-11-15 04:48:29,173] INFO : Plotted jet ratio +rank0 [2026-11-15 04:49:09,813] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:49:09,813] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:43,835] INFO : Step 19000/500000 rank0 | Train Loss: 9.0579 | Train Loss Unweighted: 2.1113 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.9515 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0698 | Regression_eta: 0.0029 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.1201 | Classification_binary: 1.9515 | Regression_cos_phi: 1.1078 | Regression_energy: 1.4299 | Regression_eta: 0.9229 | Regression_pt: 1.4664 | Regression_sin_phi: 1.0593 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0302 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3240.6 | uss_MB: 1977.6 | child_rss_MB: 23104.5 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:49:43,836] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:43,836] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:43,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:43,938] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:49:44,016] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:44,094] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:44,143] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:44,189] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:44,235] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:44,281] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:44,327] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:44,399] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:44,476] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:44,554] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:44,600] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:44,645] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:44,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:44,736] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:44,783] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:44,861] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:44,865] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:44,867] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:44,868] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:44,870] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:44,872] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:44,873] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:44,875] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:44,876] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:44,878] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:44,879] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:44,881] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:44,882] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:44,884] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:44,885] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:44,887] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:45,822] INFO : VALIDATION | Step=19000/500000 | Train Loss=9.0579 | Valid Loss=15.0260 | Train Loss Unweighted=2.1113 | Valid Loss Unweighted=2.2789 | Stale=18 | ETA=658.5m +rank0 [2026-11-15 04:49:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:46,148] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:46,152] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:46,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:46,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:46,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:46,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:47,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:47,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:47,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:47,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:47,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:47,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:48,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:48,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:48,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:49:48,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:49:48,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:49:48,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:49:49,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:49:49,176] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:49:49,183] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,567] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,796] INFO : Plotted number of elements +rank0 [2026-11-15 04:49:50,742] INFO : Plotted jets +rank0 [2026-11-15 04:49:51,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:00,040] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:50:00,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:50:40,830] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:50:40,831] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:06,049] INFO : Step 20000/500000 rank0 | Train Loss: 9.8564 | Train Loss Unweighted: 2.1169 | Train Loss Components: Classification: 0.0126 | Classification_binary: 1.9340 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0807 | Regression_eta: 0.0025 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6593 | Classification_binary: 1.9340 | Regression_cos_phi: 0.9774 | Regression_energy: 1.6539 | Regression_eta: 0.8131 | Regression_pt: 1.6377 | Regression_sin_phi: 1.1810 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2975 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3178.8 | uss_MB: 1915.8 | child_rss_MB: 23111.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:51:06,051] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:51:06,099] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:06,099] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:06,099] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:06,201] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:51:06,284] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:06,364] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:06,414] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:06,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:06,505] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:06,552] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:06,598] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:06,670] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:06,748] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:06,825] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:06,872] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:06,918] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:06,963] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:07,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:07,053] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:07,129] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:07,134] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:07,135] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:07,137] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:07,138] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:07,140] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:07,142] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:07,143] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:07,145] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:07,146] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:07,148] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:07,149] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:07,151] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:07,152] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:07,154] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:07,155] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:08,118] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.8564 | Valid Loss=14.9859 | Train Loss Unweighted=2.1169 | Valid Loss Unweighted=2.2766 | Stale=19 | ETA=657.2m +rank0 [2026-11-15 04:51:08,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:08,390] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:08,397] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:08,445] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:08,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:09,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:09,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:09,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:09,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:09,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:09,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:10,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:10,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:10,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:10,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:10,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:10,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:11,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:11,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:11,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:11,446] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:51:11,453] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:11,838] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:12,067] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:12,975] INFO : Plotted jets +rank0 [2026-11-15 04:51:14,172] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:31,617] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:51:31,617] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:13,295] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:52:13,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:29,148] INFO : Step 21000/500000 rank0 | Train Loss: 10.8404 | Train Loss Unweighted: 2.2876 | Train Loss Components: Classification: 0.0103 | Classification_binary: 2.1074 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0791 | Regression_eta: 0.0037 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3533 | Classification_binary: 2.1074 | Regression_cos_phi: 1.7273 | Regression_energy: 1.6205 | Regression_eta: 1.1942 | Regression_pt: 1.5941 | Regression_sin_phi: 1.2437 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7995s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0321 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2611 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3290.5 | uss_MB: 2027.5 | child_rss_MB: 23108.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:52:29,150] INFO : Running validation on rank0 +rank0 [2026-11-15 04:52:29,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:52:29,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:52:29,253] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:52:29,334] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:52:29,411] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:52:29,461] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:52:29,507] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:52:29,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:52:29,600] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:52:29,646] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:52:29,718] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:52:29,796] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:52:29,873] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:52:29,919] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:52:29,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:52:30,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:52:30,056] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:52:30,102] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:52:30,166] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:52:30,170] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:52:30,172] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:52:30,173] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:52:30,175] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:52:30,176] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:52:30,178] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:52:30,180] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:52:30,181] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:52:30,183] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:52:30,184] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:52:30,186] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:52:30,187] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:52:30,189] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:52:30,190] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:52:30,192] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:52:31,134] INFO : VALIDATION | Step=21000/500000 | Train Loss=10.8404 | Valid Loss=15.1942 | Train Loss Unweighted=2.2876 | Valid Loss Unweighted=2.2919 | Stale=20 | ETA=656.2m +rank0 [2026-11-15 04:52:31,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:52:31,267] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:52:31,276] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:31,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:52:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:52:32,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:52:32,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:52:32,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:52:32,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:52:32,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:52:32,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:52:32,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:52:33,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:52:33,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:52:33,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:52:33,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:52:33,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:52:33,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:52:34,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:52:34,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:52:34,346] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:52:34,365] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:34,750] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:34,979] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:35,898] INFO : Plotted jets +rank0 [2026-11-15 04:52:37,084] INFO : Plotted jet ratio +rank0 [2026-11-15 04:53:03,817] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:53:03,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:45,387] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:53:45,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:52,244] INFO : Step 22000/500000 rank0 | Train Loss: 8.8013 | Train Loss Unweighted: 1.8964 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0689 | Regression_eta: 0.0025 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7442 | Classification_binary: 1.7365 | Regression_cos_phi: 0.6952 | Regression_energy: 1.4115 | Regression_eta: 0.8193 | Regression_pt: 1.4280 | Regression_sin_phi: 0.9666 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0022 | max_backward: 0.0270 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2610 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3316.7 | uss_MB: 2053.7 | child_rss_MB: 23106.6 | allocated_MB: 170.4 | reserved_MB: 674.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:53:52,245] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:53:52,293] INFO : Running validation on rank0 +rank0 [2026-11-15 04:53:52,293] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:53:52,293] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:53:52,395] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:53:52,475] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:53:52,553] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:53:52,601] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:53:52,647] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:53:52,691] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:53:52,738] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:53:52,784] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:53:52,856] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:53:52,933] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:53:53,011] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:53:53,058] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:53:53,103] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:53:53,149] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:53:53,195] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:53:53,242] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:53:53,307] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:53:53,311] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:53:53,312] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:53:53,314] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:53:53,316] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:53:53,317] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:53:53,319] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:53:53,320] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:53:53,322] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:53:53,323] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:53:53,325] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:53:53,326] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:53:53,328] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:53:53,329] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:53:53,331] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:53:53,332] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:53:54,282] INFO : VALIDATION | Step=22000/500000 | Train Loss=8.8013 | Valid Loss=15.1373 | Train Loss Unweighted=1.8964 | Valid Loss Unweighted=2.2810 | Stale=21 | ETA=655.2m +rank0 [2026-11-15 04:53:54,282] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:53:54,413] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:53:54,421] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:54,469] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:53:55,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:53:55,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:53:55,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:53:55,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:53:55,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:53:55,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:53:56,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:53:56,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:53:56,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:53:56,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:53:56,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:53:56,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:53:56,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:53:57,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:53:57,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:53:57,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:53:57,568] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:53:57,574] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:57,963] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:58,195] INFO : Plotted number of elements +rank0 [2026-11-15 04:53:59,112] INFO : Plotted jets +rank0 [2026-11-15 04:54:00,302] INFO : Plotted jet ratio +rank0 [2026-11-15 04:54:36,188] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:54:36,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:15,327] INFO : Step 23000/500000 rank0 | Train Loss: 9.0959 | Train Loss Unweighted: 2.0937 | Train Loss Components: Classification: 0.0099 | Classification_binary: 1.9137 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0821 | Regression_eta: 0.0031 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3029 | Classification_binary: 1.9137 | Regression_cos_phi: 0.8760 | Regression_energy: 1.6837 | Regression_eta: 0.9853 | Regression_pt: 1.6271 | Regression_sin_phi: 0.7072 | LR: 4.73e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0819 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3325.3 | uss_MB: 2062.4 | child_rss_MB: 23109.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:55:15,329] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:15,329] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:15,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:15,431] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:55:15,513] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:15,591] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:15,641] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:15,687] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:15,733] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:15,780] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:15,826] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:15,898] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:15,976] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:16,052] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:16,099] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:16,145] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:16,191] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:16,237] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:16,283] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:16,347] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:16,351] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:16,353] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:16,354] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:16,356] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:16,358] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:16,359] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:16,361] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:16,362] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:16,364] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:16,365] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:16,367] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:16,368] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:16,370] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:16,372] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:16,373] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:17,332] INFO : VALIDATION | Step=23000/500000 | Train Loss=9.0959 | Valid Loss=15.2043 | Train Loss Unweighted=2.0937 | Valid Loss Unweighted=2.2998 | Stale=22 | ETA=654.1m +rank0 [2026-11-15 04:55:17,332] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:55:17,655] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:55:17,663] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:17,712] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:18,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:18,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:18,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:18,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:18,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:18,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:19,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:19,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:19,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:19,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:19,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:19,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:20,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:20,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:20,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:20,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:20,727] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:55:20,734] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:21,110] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:21,339] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:22,254] INFO : Plotted jets +rank0 [2026-11-15 04:55:23,445] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:26,925] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:55:26,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:08,412] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:56:08,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:38,160] INFO : Step 24000/500000 rank0 | Train Loss: 9.6762 | Train Loss Unweighted: 2.0494 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.8872 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0696 | Regression_eta: 0.0021 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7535 | Classification_binary: 1.8872 | Regression_cos_phi: 1.0183 | Regression_energy: 1.4263 | Regression_eta: 0.6737 | Regression_pt: 1.4179 | Regression_sin_phi: 1.4993 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7622s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0022 | max_backward: 0.0289 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2362 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3363.0 | uss_MB: 2100.0 | child_rss_MB: 23112.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:56:38,162] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:56:38,209] INFO : Running validation on rank0 +rank0 [2026-11-15 04:56:38,209] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:56:38,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:56:38,334] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:56:38,415] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:56:38,493] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:56:38,542] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:56:38,588] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:56:38,633] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:56:38,679] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:56:38,726] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:56:38,799] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:56:38,877] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:56:38,954] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:56:39,002] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:56:39,048] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:56:39,094] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:56:39,140] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:56:39,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:56:39,250] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:56:39,254] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:56:39,256] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:56:39,257] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:56:39,259] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:56:39,261] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:56:39,262] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:56:39,264] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:56:39,265] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:56:39,267] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:56:39,269] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:56:39,270] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:56:39,272] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:56:39,273] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:56:39,275] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:56:39,276] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:56:40,229] INFO : VALIDATION | Step=24000/500000 | Train Loss=9.6762 | Valid Loss=15.4143 | Train Loss Unweighted=2.0494 | Valid Loss Unweighted=2.3019 | Stale=23 | ETA=652.9m +rank0 [2026-11-15 04:56:40,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:56:40,562] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:56:40,570] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:40,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:56:41,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:56:41,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:56:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:56:41,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:56:41,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:56:41,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:56:42,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:56:42,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:56:42,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:56:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:56:42,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:56:42,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:56:43,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:56:43,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:56:43,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:56:43,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:56:43,544] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:56:43,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:43,933] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:44,161] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:45,094] INFO : Plotted jets +rank0 [2026-11-15 04:56:46,277] INFO : Plotted jet ratio +rank0 [2026-11-15 04:56:58,965] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:56:58,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:57:40,275] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:57:40,275] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:58:00,880] INFO : Step 25000/500000 rank0 | Train Loss: 8.5471 | Train Loss Unweighted: 2.1765 | Train Loss Components: Classification: 0.0110 | Classification_binary: 2.0188 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0692 | Regression_eta: 0.0024 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4539 | Classification_binary: 2.0188 | Regression_cos_phi: 0.6718 | Regression_energy: 1.4181 | Regression_eta: 0.7667 | Regression_pt: 1.4354 | Regression_sin_phi: 0.7823 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.2378 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3381.7 | uss_MB: 2118.7 | child_rss_MB: 23111.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:58:00,881] INFO : Running validation on rank0 +rank0 [2026-11-15 04:58:00,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:58:00,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:58:00,986] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:58:01,068] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:58:01,147] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:58:01,196] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:58:01,242] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:58:01,286] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:58:01,332] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:58:01,378] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:58:01,450] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:58:01,529] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:58:01,606] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:58:01,653] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:58:01,699] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:58:01,744] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:58:01,790] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:58:01,835] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:58:01,900] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:58:01,909] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:58:01,912] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:58:01,914] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:58:01,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:58:01,917] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:58:01,919] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:58:01,920] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:58:01,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:58:01,923] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:58:01,924] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:58:01,926] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:58:01,929] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:58:01,929] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:58:01,931] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:58:01,933] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:58:02,875] INFO : VALIDATION | Step=25000/500000 | Train Loss=8.5471 | Valid Loss=15.6071 | Train Loss Unweighted=2.1765 | Valid Loss Unweighted=2.3078 | Stale=24 | ETA=651.6m +rank0 [2026-11-15 04:58:02,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:58:02,993] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:58:03,001] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:03,050] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:58:03,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:58:03,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:58:03,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:58:04,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:58:04,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:58:04,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:58:04,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:58:04,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:58:04,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:58:04,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:58:05,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:58:05,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:58:05,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:58:05,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:58:05,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:58:05,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:58:06,010] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:58:06,017] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:06,396] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:06,625] INFO : Plotted number of elements +rank0 [2026-11-15 04:58:07,540] INFO : Plotted jets +rank0 [2026-11-15 04:58:08,720] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:30,102] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:58:30,102] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:11,682] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:59:11,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:23,388] INFO : Step 26000/500000 rank0 | Train Loss: 8.8150 | Train Loss Unweighted: 2.0446 | Train Loss Components: Classification: 0.0083 | Classification_binary: 1.9155 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0544 | Regression_eta: 0.0045 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.0910 | Classification_binary: 1.9155 | Regression_cos_phi: 0.9643 | Regression_energy: 1.1156 | Regression_eta: 1.4540 | Regression_pt: 1.1322 | Regression_sin_phi: 1.1425 | LR: 5.47e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8049s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2612 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3408.8 | uss_MB: 2145.8 | child_rss_MB: 23108.2 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:59:23,390] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:59:23,440] INFO : Running validation on rank0 +rank0 [2026-11-15 04:59:23,441] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:59:23,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:59:23,545] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:59:23,626] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:59:23,708] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:59:23,756] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:59:23,802] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:59:23,848] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:59:23,895] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:59:23,942] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:59:24,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:59:24,093] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:59:24,173] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:59:24,220] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:59:24,267] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:59:24,314] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:59:24,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:59:24,408] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:59:24,484] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:59:24,488] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:59:24,490] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:59:24,492] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:59:24,493] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:59:24,495] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:59:24,496] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:59:24,498] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:59:24,499] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:59:24,501] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:59:24,503] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:59:24,504] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:59:24,506] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:59:24,507] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:59:24,509] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:59:24,511] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:59:25,441] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.8150 | Valid Loss=15.6086 | Train Loss Unweighted=2.0446 | Valid Loss Unweighted=2.3190 | Stale=25 | ETA=650.4m +rank0 [2026-11-15 04:59:25,441] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:59:25,571] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:59:25,579] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:25,643] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:59:26,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:59:26,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:59:26,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:59:26,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:59:26,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:59:27,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:59:27,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:59:27,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:59:27,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:59:27,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:59:27,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:59:28,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:59:28,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:59:28,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:59:28,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:59:28,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:59:28,733] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:59:28,740] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:29,122] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:29,354] INFO : Plotted number of elements +rank0 [2026-11-15 04:59:30,769] INFO : Plotted jets +rank0 [2026-11-15 04:59:31,486] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:02,596] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 05:00:02,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:44,167] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 05:00:44,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:46,696] INFO : Step 27000/500000 rank0 | Train Loss: 10.7064 | Train Loss Unweighted: 2.3267 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.1281 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0861 | Regression_eta: 0.0030 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7824 | Classification_binary: 2.1281 | Regression_cos_phi: 1.3589 | Regression_energy: 1.7643 | Regression_eta: 0.9536 | Regression_pt: 1.8127 | Regression_sin_phi: 0.9065 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8099s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3433.2 | uss_MB: 2170.2 | child_rss_MB: 23114.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:00:46,698] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:46,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:46,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:46,800] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 05:00:46,877] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:46,953] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:47,002] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:47,048] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:47,094] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:47,140] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:47,185] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:47,258] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:47,334] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:47,410] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:47,457] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:47,503] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:47,548] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:47,594] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:47,640] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:47,716] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:47,720] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:47,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:47,724] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:47,725] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:47,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:47,728] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:47,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:47,731] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:47,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:47,735] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:47,736] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:47,738] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:47,739] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:47,741] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:47,742] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:48,678] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.7064 | Valid Loss=15.7854 | Train Loss Unweighted=2.3267 | Valid Loss Unweighted=2.3279 | Stale=26 | ETA=649.2m +rank0 [2026-11-15 05:00:48,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:00:48,994] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:00:49,001] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:49,060] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:49,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:49,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:49,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:50,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:50,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:50,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:50,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:50,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:50,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:51,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:51,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:51,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:51,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:51,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:51,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:51,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:52,050] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:00:52,068] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:52,451] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:52,679] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:53,597] INFO : Plotted jets +rank0 [2026-11-15 05:00:54,770] INFO : Plotted jet ratio +rank0 [2026-11-15 05:01:34,856] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 05:01:34,856] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:02:09,696] INFO : Step 28000/500000 rank0 | Train Loss: 9.9499 | Train Loss Unweighted: 2.1307 | Train Loss Components: Classification: 0.0103 | Classification_binary: 1.9821 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0627 | Regression_eta: 0.0051 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3512 | Classification_binary: 1.9821 | Regression_cos_phi: 1.1031 | Regression_energy: 1.2855 | Regression_eta: 1.6386 | Regression_pt: 1.2894 | Regression_sin_phi: 1.3000 | LR: 6.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7693s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0279 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3461.2 | uss_MB: 2198.2 | child_rss_MB: 23112.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 05:02:09,698] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 05:02:09,746] INFO : Running validation on rank0 +rank0 [2026-11-15 05:02:09,746] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:02:09,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:02:09,849] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 05:02:09,930] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:02:10,008] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:02:10,058] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:02:10,104] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:02:10,149] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:02:10,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:02:10,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:02:10,314] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:02:10,391] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:02:10,468] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:02:10,514] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:02:10,560] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:02:10,606] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:02:10,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:02:10,698] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:02:10,763] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:02:10,768] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:02:10,769] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:02:10,771] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:02:10,772] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:02:10,774] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:02:10,776] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:02:10,777] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:02:10,779] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:02:10,780] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:02:10,782] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:02:10,783] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:02:10,785] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:02:10,786] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:02:10,788] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:02:10,789] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:02:11,726] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.9499 | Valid Loss=15.8502 | Train Loss Unweighted=2.1307 | Valid Loss Unweighted=2.3353 | Stale=27 | ETA=648.1m +rank0 [2026-11-15 05:02:11,727] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:02:11,855] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:02:11,863] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:11,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:02:12,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:02:12,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:02:12,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:02:12,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:02:13,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:02:13,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:02:13,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:02:13,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:02:13,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:02:13,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:02:14,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:02:14,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:02:14,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:02:14,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:02:14,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:02:14,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:02:14,944] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:02:14,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:15,338] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:15,569] INFO : Plotted number of elements +rank0 [2026-11-15 05:02:16,504] INFO : Plotted jets +rank0 [2026-11-15 05:02:17,700] INFO : Plotted jet ratio +rank0 [2026-11-15 05:02:25,511] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 05:02:25,511] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:06,147] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 05:03:06,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:31,644] INFO : Step 29000/500000 rank0 | Train Loss: 10.3286 | Train Loss Unweighted: 2.1904 | Train Loss Components: Classification: 0.0091 | Classification_binary: 2.0132 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0777 | Regression_eta: 0.0027 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.2011 | Classification_binary: 2.0132 | Regression_cos_phi: 1.2094 | Regression_energy: 1.5927 | Regression_eta: 0.8769 | Regression_pt: 1.6001 | Regression_sin_phi: 1.8354 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2615 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3488.2 | uss_MB: 2225.2 | child_rss_MB: 23109.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:03:31,646] INFO : Running validation on rank0 +rank0 [2026-11-15 05:03:31,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:03:31,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:03:31,752] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 05:03:31,833] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:03:31,912] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:03:31,962] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:03:32,008] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:03:32,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:03:32,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:03:32,151] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:03:32,223] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:03:32,301] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:03:32,380] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:03:32,427] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:03:32,473] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:03:32,519] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:03:32,565] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:03:32,612] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:03:32,678] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:03:32,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:03:32,684] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:03:32,686] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:03:32,688] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:03:32,689] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:03:32,691] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:03:32,693] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:03:32,694] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:03:32,696] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:03:32,697] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:03:32,699] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:03:32,700] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:03:32,702] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:03:32,704] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:03:32,705] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:03:33,643] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.3286 | Valid Loss=15.9546 | Train Loss Unweighted=2.1904 | Valid Loss Unweighted=2.3392 | Stale=28 | ETA=646.6m +rank0 [2026-11-15 05:03:33,643] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:03:33,768] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:03:33,777] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:33,846] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:03:34,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:03:34,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:03:34,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:03:34,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:03:35,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:03:35,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:03:35,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:03:35,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:03:35,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:03:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:03:35,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:03:36,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:03:36,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:03:36,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:03:36,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:03:36,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:03:36,868] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:03:36,875] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:37,258] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:37,492] INFO : Plotted number of elements +rank0 [2026-11-15 05:03:38,426] INFO : Plotted jets +rank0 [2026-11-15 05:03:39,619] INFO : Plotted jet ratio +rank0 [2026-11-15 05:03:56,505] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 05:03:56,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:04:37,921] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 05:04:37,921] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:04:54,503] INFO : Step 30000/500000 rank0 | Train Loss: 10.5399 | Train Loss Unweighted: 2.0270 | Train Loss Components: Classification: 0.0111 | Classification_binary: 1.8312 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0864 | Regression_eta: 0.0022 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4651 | Classification_binary: 1.8312 | Regression_cos_phi: 0.6968 | Regression_energy: 1.7715 | Regression_eta: 0.7114 | Regression_pt: 1.7713 | Regression_sin_phi: 2.2925 | LR: 6.58e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8446s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0271 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2646 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3511.9 | uss_MB: 2248.9 | child_rss_MB: 23103.4 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:04:54,505] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 05:04:54,555] INFO : Running validation on rank0 +rank0 [2026-11-15 05:04:54,555] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:04:54,555] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:04:54,657] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 05:04:54,737] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:04:54,815] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:04:54,865] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:04:54,911] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:04:54,957] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:04:55,003] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:04:55,050] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:04:55,123] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:04:55,202] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:04:55,279] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:04:55,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:04:55,374] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:04:55,420] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:04:55,467] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:04:55,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:04:55,578] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:04:55,582] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:04:55,583] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:04:55,585] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:04:55,587] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:04:55,588] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:04:55,590] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:04:55,591] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:04:55,593] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:04:55,595] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:04:55,596] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:04:55,598] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:04:55,599] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:04:55,601] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:04:55,602] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:04:55,604] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:04:56,543] INFO : VALIDATION | Step=30000/500000 | Train Loss=10.5399 | Valid Loss=16.1117 | Train Loss Unweighted=2.0270 | Valid Loss Unweighted=2.3436 | Stale=29 | ETA=645.3m +rank0 [2026-11-15 05:04:56,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:04:56,925] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:04:56,941] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:04:57,008] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:57,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:57,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:58,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:58,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:58,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:58,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:04:58,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:04:58,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:04:58,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:04:59,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:04:59,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:04:59,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:04:59,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:04:59,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:04:59,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:00,017] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:05:00,024] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:00,400] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:00,632] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:01,556] INFO : Plotted jets +rank0 [2026-11-15 05:05:02,767] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:28,800] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 05:05:28,801] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:10,369] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 05:06:10,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:17,735] INFO : Step 31000/500000 rank0 | Train Loss: 10.0723 | Train Loss Unweighted: 2.0259 | Train Loss Components: Classification: 0.0096 | Classification_binary: 1.8785 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0623 | Regression_eta: 0.0054 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.2658 | Classification_binary: 1.8785 | Regression_cos_phi: 0.9365 | Regression_energy: 1.2774 | Regression_eta: 1.7380 | Regression_pt: 1.2664 | Regression_sin_phi: 1.7097 | LR: 6.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0279 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2535 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3449.6 | uss_MB: 2186.6 | child_rss_MB: 23105.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 05:06:17,737] INFO : Running validation on rank0 +rank0 [2026-11-15 05:06:17,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:06:17,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:06:17,840] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 05:06:17,924] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:06:18,002] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:06:18,054] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:06:18,101] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:06:18,148] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:06:18,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:06:18,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:06:18,314] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:06:18,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:06:18,471] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:06:18,517] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:06:18,564] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:06:18,610] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:06:18,657] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:06:18,703] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:06:18,768] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:06:18,772] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:06:18,773] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:06:18,775] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:06:18,777] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:06:18,778] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:06:18,780] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:06:18,781] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:06:18,783] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:06:18,784] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:06:18,786] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:06:18,787] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:06:18,789] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:06:18,790] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:06:18,792] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:06:18,794] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:06:19,757] INFO : VALIDATION | Step=31000/500000 | Train Loss=10.0723 | Valid Loss=16.3077 | Train Loss Unweighted=2.0259 | Valid Loss Unweighted=2.3535 | Stale=30 | ETA=644.2m +rank0 [2026-11-15 05:06:19,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:06:19,872] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:06:19,880] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:19,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:06:20,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:06:20,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:06:20,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:06:20,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:06:21,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:06:21,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:06:21,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:06:21,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:06:21,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:06:21,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:06:22,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:06:22,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:06:22,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:06:22,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:06:22,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:06:22,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:06:22,909] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:06:22,915] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:23,309] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:23,546] INFO : Plotted number of elements +rank0 [2026-11-15 05:06:24,481] INFO : Plotted jets +rank0 [2026-11-15 05:06:25,715] INFO : Plotted jet ratio +rank0 [2026-11-15 05:06:26,098] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 05:06:26,098] INFO : Training completed. Total time on device 0: 42.684min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..aaaf841f03130858d9f21e996c1d6a2dfd80a42b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/train.log.0 @@ -0,0 +1,2495 @@ +rank0 [2026-11-15 04:23:43,955] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-15 04:23:43,955] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-15 04:23:43,955] INFO : Instantiating model +rank0 [2026-11-15 04:23:43,973] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-15 04:23:43,973] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-15 04:23:43,974] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-15 04:23:43,975] INFO : nn0 parameters: 140800 +rank0 [2026-11-15 04:23:43,975] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-15 04:23:43,975] INFO : Initializing backbone layer 0 +rank0 [2026-11-15 04:23:43,975] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,977] INFO : Initializing backbone layer 1 +rank0 [2026-11-15 04:23:43,977] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,979] INFO : Initializing backbone layer 2 +rank0 [2026-11-15 04:23:43,979] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,981] INFO : Initializing backbone layer 3 +rank0 [2026-11-15 04:23:43,981] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,983] INFO : Initializing backbone layer 4 +rank0 [2026-11-15 04:23:43,983] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,985] INFO : Initializing backbone layer 5 +rank0 [2026-11-15 04:23:43,985] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,988] INFO : Backbone initialization took 0.01s +rank0 [2026-11-15 04:23:43,988] INFO : backbone parameters: 2374656 +rank0 [2026-11-15 04:23:43,988] INFO : Initializing output DNNs +rank0 [2026-11-15 04:23:43,989] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,991] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-15 04:23:43,997] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-15 04:23:43,997] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-15 04:23:43,997] INFO : jagged_attention enabled=True +rank0 [2026-11-15 04:23:43,997] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-15 04:23:43,997] INFO : task_queries enabled=True +rank0 [2026-11-15 04:23:43,998] INFO : classification_norm parameters: 512 +rank0 [2026-11-15 04:23:43,998] INFO : regression_norm parameters: 512 +rank0 [2026-11-15 04:23:43,998] INFO : classification_query parameters: 256 +rank0 [2026-11-15 04:23:43,998] INFO : regression_query parameters: 256 +rank0 [2026-11-15 04:23:43,998] INFO : classification_readout parameters: 396288 +rank0 [2026-11-15 04:23:43,998] INFO : regression_readout parameters: 396288 +rank0 [2026-11-15 04:23:43,998] INFO : output_mode=elementwise +rank0 [2026-11-15 04:23:43,998] INFO : set_decoder parameters: 0 +rank0 [2026-11-15 04:23:43,998] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_pid parameters: 67846 +rank0 [2026-11-15 04:23:43,998] INFO : nn_pt parameters: 133122 +rank0 [2026-11-15 04:23:43,998] INFO : nn_eta parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-15 04:23:43,998] INFO : nn_energy parameters: 133122 +rank0 [2026-11-15 04:23:43,999] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-15 04:23:43,999] INFO : MLPF __init__ done +rank0 [2026-11-15 04:23:43,999] INFO : Instantiated model +rank0 [2026-11-15 04:23:44,000] INFO : Moving model to device rank=0 +rank0 [2026-11-15 04:23:44,006] INFO : Moved model to device rank=0 +rank0 [2026-11-15 04:23:44,006] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-15 04:23:44,080] INFO : Reset task loss weights for calibration on the fine-tuning data +rank0 [2026-11-15 04:23:44,080] INFO : Loaded only the model weights, starting training from step 1 +rank0 [2026-11-15 04:23:44,081] INFO : Loaded model weights from experiments/pf_fm_joint/joint_seed12345_20261114_063930_566389/checkpoints/checkpoint-500000.pth +rank0 [2026-11-15 04:23:44,139] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-15 04:23:44,139] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-15 04:23:44,140] INFO : Trainable parameters: 3910930 +rank0 [2026-11-15 04:23:44,140] INFO : Non-trainable parameters: 0 +rank0 [2026-11-15 04:23:44,140] INFO : Total parameters: 3910930 +rank0 [2026-11-15 04:23:44,142] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-15 04:23:44,143] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334 +rank0 [2026-11-15 04:23:44,143] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334 +rank0 [2026-11-15 04:23:44,143] INFO : Getting dataloaders +rank0 [2026-11-15 04:23:44,143] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:44,303] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,351] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,387] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,428] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,468] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,533] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,585] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,632] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,670] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,710] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-15 04:23:44,710] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-15 04:23:44,710] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-15 04:23:44,710] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-15 04:23:44,710] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:23:44,723] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,732] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,744] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,757] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,770] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,781] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,792] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,803] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,817] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,828] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-15 04:23:44,829] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-15 04:23:44,829] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-15 04:23:44,859] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:45,030] INFO : Got dataloaders +rank0 [2026-11-15 04:23:45,031] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-15 04:23:45,031] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-15 04:23:45,031] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-15 04:23:45,038] INFO : Creating train_iterator +rank0 [2026-11-15 04:23:45,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:23:45,069] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:23:45,277] INFO : Created train_iterator +rank0 [2026-11-15 04:23:53,626] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 1.9723292076587677, 'Classification': 0.014986821804195643, 'Regression_pt': 0.09762214239686727, 'Regression_eta': 0.006117531423224136, 'Regression_sin_phi': 0.0055219064909033475, 'Regression_cos_phi': 0.005466182068921625, 'Regression_energy': 0.09622789811342955}, weights={'Classification_binary': 1.0, 'Classification': 131.60423278808594, 'Regression_pt': 20.203706741333008, 'Regression_eta': 322.40606689453125, 'Regression_sin_phi': 357.1826477050781, 'Regression_cos_phi': 360.82391357421875, 'Regression_energy': 20.49643898010254} +rank0 [2026-11-15 04:24:27,826] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-15 04:24:27,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:00,125] INFO : Step 1000/500000 rank0 | Train Loss: 14.1385 | Train Loss Unweighted: 2.3414 | Train Loss Components: Classification: 0.0160 | Classification_binary: 2.0702 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1199 | Regression_eta: 0.0035 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.1109 | Classification_binary: 2.0702 | Regression_cos_phi: 2.0246 | Regression_energy: 2.4574 | Regression_eta: 1.1386 | Regression_pt: 2.4412 | Regression_sin_phi: 1.8957 | LR: 2.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7677s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0277 | max_optimizer_step: 0.0805 | max_tensorboard_logging: 0.2640 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2622.9 | uss_MB: 1351.1 | child_rss_MB: 23087.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:25:00,126] INFO : Running validation on rank0 +rank0 [2026-11-15 04:25:00,126] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:25:00,127] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:25:00,251] INFO : saved batch0_epoch1000.parquet +rank0 [2026-11-15 04:25:00,370] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:25:00,426] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:25:00,464] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:25:00,503] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:25:00,536] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:25:00,575] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:25:00,613] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:25:00,649] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:25:00,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:25:00,756] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:25:00,794] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:25:00,832] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:25:00,866] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:25:00,903] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:25:00,942] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:25:00,978] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:25:00,981] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:25:00,983] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:25:00,984] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:25:00,986] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:25:00,987] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:25:00,989] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:25:00,990] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:25:00,992] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:25:00,994] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:25:00,995] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:25:00,997] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:25:00,998] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:25:01,000] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:25:01,001] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:25:01,003] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:25:02,014] INFO : VALIDATION | Step=1000/500000 | Train Loss=14.1385 | Valid Loss=14.1806 | Train Loss Unweighted=2.3414 | Valid Loss Unweighted=2.2025 | Stale=0 | ETA=640.2m +rank0 [2026-11-15 04:25:02,015] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:25:02,501] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:25:02,509] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:02,747] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:25:03,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:25:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:25:03,513] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:25:03,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:25:03,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:25:03,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:25:04,156] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:25:04,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:25:04,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:25:04,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:25:04,791] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:25:04,952] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:25:05,123] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:25:05,284] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:25:05,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:25:05,598] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:25:05,705] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:25:05,712] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:06,092] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_1000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:25:06,381] INFO : Plotted number of elements +rank0 [2026-11-15 04:25:07,332] INFO : Plotted jets +rank0 [2026-11-15 04:25:08,454] INFO : Plotted jet ratio +rank0 [2026-11-15 04:25:17,892] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-15 04:25:17,892] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:25:59,163] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-15 04:25:59,163] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:26:22,613] INFO : Step 2000/500000 rank0 | Train Loss: 13.4749 | Train Loss Unweighted: 2.3646 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.1262 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1031 | Regression_eta: 0.0040 | Regression_pt: 0.1027 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.5403 | Classification_binary: 2.1262 | Regression_cos_phi: 1.7725 | Regression_energy: 2.1129 | Regression_eta: 1.2971 | Regression_pt: 2.0741 | Regression_sin_phi: 1.5518 | LR: 2.02e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8093s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0236 | max_loss: 0.0021 | max_backward: 0.0258 | max_optimizer_step: 0.0835 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2777.4 | uss_MB: 1514.0 | child_rss_MB: 23090.9 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:26:22,615] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-2000.pth +rank0 [2026-11-15 04:26:22,665] INFO : Running validation on rank0 +rank0 [2026-11-15 04:26:22,665] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:26:22,665] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:26:22,770] INFO : saved batch0_epoch2000.parquet +rank0 [2026-11-15 04:26:22,850] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:26:22,926] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:26:22,973] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:26:23,018] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:26:23,063] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:26:23,109] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:26:23,154] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:26:23,225] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:26:23,302] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:26:23,378] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:26:23,426] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:26:23,471] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:26:23,516] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:26:23,560] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:26:23,605] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:26:23,681] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:26:23,686] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:26:23,687] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:26:23,689] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:26:23,691] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:26:23,692] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:26:23,694] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:26:23,695] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:26:23,698] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:26:23,698] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:26:23,700] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:26:23,701] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:26:23,703] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:26:23,705] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:26:23,706] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:26:23,708] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:26:24,662] INFO : VALIDATION | Step=2000/500000 | Train Loss=13.4749 | Valid Loss=14.2400 | Train Loss Unweighted=2.3646 | Valid Loss Unweighted=2.2050 | Stale=1 | ETA=662.4m +rank0 [2026-11-15 04:26:24,662] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:26:24,814] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:26:24,821] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:24,867] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:26:25,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:26:25,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:26:25,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:26:25,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:26:26,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:26:26,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:26:26,351] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:26:26,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:26:26,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:26:26,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:26:26,983] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:26:27,141] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:26:27,297] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:26:27,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:26:27,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:26:27,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:26:27,846] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:26:27,854] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:28,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_2000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:26:28,464] INFO : Plotted number of elements +rank0 [2026-11-15 04:26:29,801] INFO : Plotted jets +rank0 [2026-11-15 04:26:30,514] INFO : Plotted jet ratio +rank0 [2026-11-15 04:26:48,903] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-15 04:26:48,903] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:30,158] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-15 04:27:30,158] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:27:44,540] INFO : Step 3000/500000 rank0 | Train Loss: 11.8225 | Train Loss Unweighted: 2.0055 | Train Loss Components: Classification: 0.0171 | Classification_binary: 1.8118 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0802 | Regression_eta: 0.0061 | Regression_pt: 0.0834 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.2449 | Classification_binary: 1.8118 | Regression_cos_phi: 1.3865 | Regression_energy: 1.6447 | Regression_eta: 1.9632 | Regression_pt: 1.6852 | Regression_sin_phi: 1.0861 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7470s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0265 | max_optimizer_step: 0.0792 | max_tensorboard_logging: 0.2396 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2834.5 | uss_MB: 1571.3 | child_rss_MB: 23093.8 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:27:44,541] INFO : Running validation on rank0 +rank0 [2026-11-15 04:27:44,541] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:27:44,541] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:27:44,643] INFO : saved batch0_epoch3000.parquet +rank0 [2026-11-15 04:27:44,724] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:27:44,801] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:27:44,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:27:44,894] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:27:44,940] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:27:44,987] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:27:45,034] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:27:45,105] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:27:45,182] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:27:45,259] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:27:45,307] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:27:45,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:27:45,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:27:45,444] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:27:45,490] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:27:45,566] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:27:45,571] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:27:45,573] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:27:45,574] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:27:45,576] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:27:45,578] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:27:45,579] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:27:45,581] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:27:45,582] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:27:45,584] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:27:45,586] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:27:45,587] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:27:45,589] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:27:45,590] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:27:45,592] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:27:45,593] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:27:46,534] INFO : VALIDATION | Step=3000/500000 | Train Loss=11.8225 | Valid Loss=14.2182 | Train Loss Unweighted=2.0055 | Valid Loss Unweighted=2.2061 | Stale=2 | ETA=666.8m +rank0 [2026-11-15 04:27:46,534] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:27:46,983] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:27:47,002] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:47,059] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:27:47,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:27:47,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:27:47,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:27:47,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:27:48,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:27:48,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:27:48,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:27:48,639] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:27:48,802] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:27:48,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:27:49,127] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:27:49,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:27:49,451] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:27:49,614] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:27:49,778] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:27:49,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:27:49,987] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:27:49,994] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:50,374] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_3000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:27:50,601] INFO : Plotted number of elements +rank0 [2026-11-15 04:27:51,972] INFO : Plotted jets +rank0 [2026-11-15 04:27:52,679] INFO : Plotted jet ratio +rank0 [2026-11-15 04:28:20,149] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-15 04:28:20,149] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:01,035] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-15 04:29:01,035] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:29:06,628] INFO : Step 4000/500000 rank0 | Train Loss: 13.8515 | Train Loss Unweighted: 2.0156 | Train Loss Components: Classification: 0.0156 | Classification_binary: 1.8005 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0886 | Regression_eta: 0.0048 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.0561 | Classification_binary: 1.8005 | Regression_cos_phi: 2.6462 | Regression_energy: 1.8167 | Regression_eta: 1.5528 | Regression_pt: 1.8747 | Regression_sin_phi: 2.1044 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8267s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2918 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2868.9 | uss_MB: 1605.8 | child_rss_MB: 23098.3 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:29:06,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-4000.pth +rank0 [2026-11-15 04:29:06,679] INFO : Running validation on rank0 +rank0 [2026-11-15 04:29:06,679] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:29:06,679] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:29:06,780] INFO : saved batch0_epoch4000.parquet +rank0 [2026-11-15 04:29:06,859] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:29:06,936] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:29:06,983] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:29:07,029] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:29:07,074] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:29:07,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:29:07,166] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:29:07,237] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:29:07,312] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:29:07,389] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:29:07,435] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:29:07,479] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:29:07,524] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:29:07,570] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:29:07,615] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:29:07,679] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:29:07,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:29:07,685] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:29:07,688] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:29:07,689] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:29:07,690] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:29:07,692] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:29:07,694] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:29:07,695] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:29:07,698] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:29:07,698] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:29:07,700] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:29:07,702] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:29:07,704] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:29:07,705] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:29:07,707] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:29:08,674] INFO : VALIDATION | Step=4000/500000 | Train Loss=13.8515 | Valid Loss=14.2494 | Train Loss Unweighted=2.0156 | Valid Loss Unweighted=2.2104 | Stale=3 | ETA=668.8m +rank0 [2026-11-15 04:29:08,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:29:08,806] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:29:08,814] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:08,860] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:29:09,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:29:09,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:29:09,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:29:09,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:29:10,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:29:10,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:29:10,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:29:10,537] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:29:10,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:29:10,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:29:11,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:29:11,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:29:11,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:29:11,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:29:11,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:29:11,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:29:11,883] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:29:11,890] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:12,274] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_4000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:29:12,503] INFO : Plotted number of elements +rank0 [2026-11-15 04:29:13,848] INFO : Plotted jets +rank0 [2026-11-15 04:29:14,563] INFO : Plotted jet ratio +rank0 [2026-11-15 04:29:51,200] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-15 04:29:51,200] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:30:28,843] INFO : Step 5000/500000 rank0 | Train Loss: 11.0816 | Train Loss Unweighted: 2.0197 | Train Loss Components: Classification: 0.0123 | Classification_binary: 1.8056 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0942 | Regression_eta: 0.0023 | Regression_pt: 0.0969 | Regression_sin_phi: 0.0021 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6142 | Classification_binary: 1.8056 | Regression_cos_phi: 2.2568 | Regression_energy: 1.9301 | Regression_eta: 0.7508 | Regression_pt: 1.9576 | Regression_sin_phi: 0.7665 | LR: 2.13e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0790 | max_tensorboard_logging: 0.2270 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2888.7 | uss_MB: 1625.7 | child_rss_MB: 23097.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:30:28,845] INFO : Running validation on rank0 +rank0 [2026-11-15 04:30:28,845] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:30:28,845] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:30:28,946] INFO : saved batch0_epoch5000.parquet +rank0 [2026-11-15 04:30:29,024] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:30:29,101] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:30:29,149] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:30:29,194] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:30:29,238] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:30:29,284] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:30:29,331] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:30:29,402] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:30:29,478] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:30:29,554] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:30:29,601] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:30:29,646] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:30:29,691] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:30:29,739] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:30:29,785] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:30:29,850] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:30:29,854] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:30:29,856] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:30:29,858] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:30:29,859] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:30:29,861] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:30:29,862] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:30:29,864] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:30:29,865] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:30:29,867] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:30:29,869] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:30:29,870] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:30:29,872] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:30:29,873] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:30:29,875] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:30:29,876] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:30:30,813] INFO : VALIDATION | Step=5000/500000 | Train Loss=11.0816 | Valid Loss=14.3128 | Train Loss Unweighted=2.0197 | Valid Loss Unweighted=2.2150 | Stale=4 | ETA=669.5m +rank0 [2026-11-15 04:30:30,814] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:30:31,199] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:30:31,206] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:31,264] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:30:31,740] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:30:31,908] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:30:32,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:30:32,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:30:32,386] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:30:32,544] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:30:32,704] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:30:32,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:30:33,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:30:33,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:30:33,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:30:33,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:30:33,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:30:33,846] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:30:34,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:30:34,154] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:30:34,241] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:30:34,247] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:34,628] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_5000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:30:34,855] INFO : Plotted number of elements +rank0 [2026-11-15 04:30:35,777] INFO : Plotted jets +rank0 [2026-11-15 04:30:36,919] INFO : Plotted jet ratio +rank0 [2026-11-15 04:30:41,483] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-15 04:30:41,483] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:22,628] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-15 04:31:22,628] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:31:51,141] INFO : Step 6000/500000 rank0 | Train Loss: 12.4790 | Train Loss Unweighted: 2.2422 | Train Loss Components: Classification: 0.0123 | Classification_binary: 2.0500 | Regression_cos_phi: 0.0053 | Regression_energy: 0.0811 | Regression_eta: 0.0052 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6178 | Classification_binary: 2.0500 | Regression_cos_phi: 1.9294 | Regression_energy: 1.6616 | Regression_eta: 1.6766 | Regression_pt: 1.6798 | Regression_sin_phi: 1.8638 | LR: 2.19e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7855s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0794 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2832.1 | uss_MB: 1569.1 | child_rss_MB: 23098.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 04:31:51,143] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-6000.pth +rank0 [2026-11-15 04:31:51,206] INFO : Running validation on rank0 +rank0 [2026-11-15 04:31:51,206] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:31:51,206] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:31:51,309] INFO : saved batch0_epoch6000.parquet +rank0 [2026-11-15 04:31:51,392] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:31:51,470] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:31:51,519] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:31:51,565] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:31:51,610] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:31:51,656] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:31:51,702] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:31:51,773] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:31:51,851] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:31:51,927] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:31:51,973] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:31:52,019] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:31:52,063] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:31:52,108] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:31:52,154] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:31:52,230] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:31:52,234] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:31:52,236] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:31:52,237] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:31:52,239] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:31:52,241] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:31:52,242] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:31:52,244] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:31:52,245] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:31:52,247] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:31:52,248] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:31:52,250] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:31:52,251] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:31:52,253] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:31:52,254] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:31:52,256] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:31:53,220] INFO : VALIDATION | Step=6000/500000 | Train Loss=12.4790 | Valid Loss=14.2696 | Train Loss Unweighted=2.2422 | Valid Loss Unweighted=2.2203 | Stale=5 | ETA=669.9m +rank0 [2026-11-15 04:31:53,220] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:31:53,548] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:31:53,554] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:53,602] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:31:54,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:31:54,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:31:54,356] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:31:54,522] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:31:54,683] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:31:54,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:31:55,004] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:31:55,164] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:31:55,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:31:55,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:31:55,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:31:55,808] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:31:55,968] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:31:56,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:31:56,288] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:31:56,434] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:31:56,526] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:31:56,533] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:56,917] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_6000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:31:57,145] INFO : Plotted number of elements +rank0 [2026-11-15 04:31:58,529] INFO : Plotted jets +rank0 [2026-11-15 04:31:59,249] INFO : Plotted jet ratio +rank0 [2026-11-15 04:32:13,009] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-15 04:32:13,009] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:32:54,391] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-15 04:32:54,391] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:33:13,978] INFO : Step 7000/500000 rank0 | Train Loss: 12.2047 | Train Loss Unweighted: 2.1742 | Train Loss Components: Classification: 0.0165 | Classification_binary: 1.9809 | Regression_cos_phi: 0.0032 | Regression_energy: 0.0816 | Regression_eta: 0.0063 | Regression_pt: 0.0814 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.1735 | Classification_binary: 1.9809 | Regression_cos_phi: 1.1710 | Regression_energy: 1.6722 | Regression_eta: 2.0331 | Regression_pt: 1.6451 | Regression_sin_phi: 1.5290 | LR: 2.26e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8139s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0259 | max_optimizer_step: 0.0831 | max_tensorboard_logging: 0.2938 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2942.3 | uss_MB: 1679.3 | child_rss_MB: 23101.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:33:13,979] INFO : Running validation on rank0 +rank0 [2026-11-15 04:33:13,979] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:33:13,979] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:33:14,087] INFO : saved batch0_epoch7000.parquet +rank0 [2026-11-15 04:33:14,169] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:33:14,246] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:33:14,294] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:33:14,339] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:33:14,385] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:33:14,431] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,478] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:33:14,550] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:33:14,626] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:33:14,703] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:33:14,749] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:33:14,794] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:33:14,838] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:33:14,884] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:33:14,930] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:33:15,007] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:33:15,012] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:33:15,013] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:33:15,015] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:33:15,016] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:33:15,018] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:33:15,020] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:33:15,021] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:33:15,023] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:33:15,024] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:33:15,026] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:33:15,027] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:33:15,029] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:33:15,030] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:33:15,032] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:33:15,034] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:33:16,000] INFO : VALIDATION | Step=7000/500000 | Train Loss=12.2047 | Valid Loss=14.3210 | Train Loss Unweighted=2.1742 | Valid Loss Unweighted=2.2252 | Stale=6 | ETA=670.2m +rank0 [2026-11-15 04:33:16,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:33:16,381] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:33:16,385] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:16,451] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:33:16,933] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:33:17,098] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:33:17,260] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:33:17,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:33:17,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:33:17,749] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:33:17,910] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:33:18,068] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:33:18,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:33:18,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:33:18,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:33:18,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:33:18,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:33:19,050] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:33:19,214] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:33:19,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:33:19,446] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:33:19,452] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:19,835] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_7000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:33:20,065] INFO : Plotted number of elements +rank0 [2026-11-15 04:33:21,439] INFO : Plotted jets +rank0 [2026-11-15 04:33:22,146] INFO : Plotted jet ratio +rank0 [2026-11-15 04:33:44,770] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-15 04:33:44,770] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:26,222] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-15 04:34:26,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:34:36,620] INFO : Step 8000/500000 rank0 | Train Loss: 13.6201 | Train Loss Unweighted: 2.1452 | Train Loss Components: Classification: 0.0148 | Classification_binary: 1.9558 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0782 | Regression_eta: 0.0088 | Regression_pt: 0.0772 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.9478 | Classification_binary: 1.9558 | Regression_cos_phi: 2.1029 | Regression_energy: 1.6035 | Regression_eta: 2.8346 | Regression_pt: 1.5605 | Regression_sin_phi: 1.6149 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7494s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0797 | max_tensorboard_logging: 0.2423 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.4 | uss_MB: 1698.4 | child_rss_MB: 23098.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:34:36,621] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-8000.pth +rank0 [2026-11-15 04:34:36,672] INFO : Running validation on rank0 +rank0 [2026-11-15 04:34:36,672] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:34:36,672] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:34:36,774] INFO : saved batch0_epoch8000.parquet +rank0 [2026-11-15 04:34:36,854] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:34:36,931] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:34:36,980] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:34:37,025] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:34:37,071] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:34:37,117] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:34:37,162] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:34:37,234] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:34:37,311] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:34:37,387] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:34:37,433] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:34:37,478] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:34:37,523] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:34:37,569] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:34:37,615] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:34:37,678] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:34:37,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:34:37,684] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:34:37,686] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:34:37,687] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:34:37,689] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:34:37,690] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:34:37,692] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:34:37,693] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:34:37,695] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:34:37,697] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:34:37,698] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:34:37,700] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:34:37,701] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:34:37,703] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:34:37,704] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:34:38,675] INFO : VALIDATION | Step=8000/500000 | Train Loss=13.6201 | Valid Loss=14.3575 | Train Loss Unweighted=2.1452 | Valid Loss Unweighted=2.2211 | Stale=7 | ETA=670.0m +rank0 [2026-11-15 04:34:38,675] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:34:38,798] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:34:38,804] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:38,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:34:39,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:34:39,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:34:39,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:34:39,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:34:39,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:34:40,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:34:40,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:34:40,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:34:40,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:34:40,765] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:34:40,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:34:41,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:34:41,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:34:41,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:34:41,563] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:34:41,715] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:34:41,791] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:34:41,798] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:42,174] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_8000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:34:42,406] INFO : Plotted number of elements +rank0 [2026-11-15 04:34:43,776] INFO : Plotted jets +rank0 [2026-11-15 04:34:44,501] INFO : Plotted jet ratio +rank0 [2026-11-15 04:35:16,277] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-15 04:35:16,277] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:56,995] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-15 04:35:56,996] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:35:58,271] INFO : Step 9000/500000 rank0 | Train Loss: 11.4107 | Train Loss Unweighted: 2.1223 | Train Loss Components: Classification: 0.0105 | Classification_binary: 1.9463 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0748 | Regression_eta: 0.0050 | Regression_pt: 0.0763 | Regression_sin_phi: 0.0029 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3837 | Classification_binary: 1.9463 | Regression_cos_phi: 2.3511 | Regression_energy: 1.5331 | Regression_eta: 1.6242 | Regression_pt: 1.5406 | Regression_sin_phi: 1.0317 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7573s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2907.2 | uss_MB: 1644.3 | child_rss_MB: 23101.5 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:35:58,272] INFO : Running validation on rank0 +rank0 [2026-11-15 04:35:58,273] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:35:58,273] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:35:58,374] INFO : saved batch0_epoch9000.parquet +rank0 [2026-11-15 04:35:58,456] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:35:58,532] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:35:58,581] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:35:58,627] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:35:58,672] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:35:58,718] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:35:58,764] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:35:58,835] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:35:58,916] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:35:58,992] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:35:59,039] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:35:59,084] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:35:59,129] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:35:59,175] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:35:59,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:35:59,297] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:35:59,302] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:35:59,305] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:35:59,306] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:35:59,307] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:35:59,310] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:35:59,310] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:35:59,312] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:35:59,315] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:35:59,315] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:35:59,317] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:35:59,320] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:35:59,320] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:35:59,322] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:35:59,324] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:35:59,325] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:36:00,276] INFO : VALIDATION | Step=9000/500000 | Train Loss=11.4107 | Valid Loss=14.3359 | Train Loss Unweighted=2.1223 | Valid Loss Unweighted=2.2272 | Stale=8 | ETA=668.5m +rank0 [2026-11-15 04:36:00,277] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:36:00,598] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:36:00,602] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:00,659] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:36:01,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:36:01,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:36:01,431] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:36:01,594] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:36:01,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:36:01,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:36:02,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:36:02,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:36:02,400] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:36:02,562] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:36:02,724] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:36:02,884] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:36:03,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:36:03,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:36:03,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:36:03,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:36:03,641] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:36:03,647] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:04,033] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_9000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:36:04,262] INFO : Plotted number of elements +rank0 [2026-11-15 04:36:05,181] INFO : Plotted jets +rank0 [2026-11-15 04:36:06,347] INFO : Plotted jet ratio +rank0 [2026-11-15 04:36:47,227] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-15 04:36:47,227] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:37:20,744] INFO : Step 10000/500000 rank0 | Train Loss: 10.2994 | Train Loss Unweighted: 1.7760 | Train Loss Components: Classification: 0.0129 | Classification_binary: 1.6080 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0725 | Regression_eta: 0.0041 | Regression_pt: 0.0708 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7006 | Classification_binary: 1.6080 | Regression_cos_phi: 1.3247 | Regression_energy: 1.4856 | Regression_eta: 1.3133 | Regression_pt: 1.4298 | Regression_sin_phi: 1.4373 | LR: 2.52e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7324s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0265 | max_optimizer_step: 0.0812 | max_tensorboard_logging: 0.2185 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3021.4 | uss_MB: 1758.4 | child_rss_MB: 23102.8 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:37:20,745] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-10000.pth +rank0 [2026-11-15 04:37:20,793] INFO : Running validation on rank0 +rank0 [2026-11-15 04:37:20,793] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:37:20,793] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:37:20,898] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-15 04:37:20,979] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:37:21,057] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:37:21,106] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:37:21,151] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:37:21,197] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:37:21,244] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:37:21,291] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:37:21,364] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:37:21,441] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:37:21,518] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:37:21,564] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:37:21,609] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:37:21,653] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:37:21,699] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,744] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:37:21,808] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:37:21,812] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:37:21,814] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:37:21,815] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:37:21,817] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:37:21,819] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:37:21,820] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:37:21,822] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:37:21,823] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:37:21,825] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:37:21,826] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:37:21,828] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:37:21,829] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:37:21,831] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:37:21,833] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:37:21,834] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:37:22,793] INFO : VALIDATION | Step=10000/500000 | Train Loss=10.2994 | Valid Loss=14.3647 | Train Loss Unweighted=1.7760 | Valid Loss Unweighted=2.2301 | Stale=9 | ETA=667.8m +rank0 [2026-11-15 04:37:22,793] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:37:22,923] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:37:22,927] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:22,975] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:37:23,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:37:23,566] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:37:23,725] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:37:23,889] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:37:24,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:37:24,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:37:24,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:37:24,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:37:24,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:37:24,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:37:25,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:37:25,193] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:37:25,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:37:25,512] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:37:25,673] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:37:25,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:37:25,920] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:37:25,927] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,318] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:37:26,553] INFO : Plotted number of elements +rank0 [2026-11-15 04:37:27,469] INFO : Plotted jets +rank0 [2026-11-15 04:37:28,646] INFO : Plotted jet ratio +rank0 [2026-11-15 04:37:37,437] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-15 04:37:37,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:18,468] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-15 04:38:18,469] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:38:42,562] INFO : Step 11000/500000 rank0 | Train Loss: 21.5883 | Train Loss Unweighted: 2.4287 | Train Loss Components: Classification: 0.0122 | Classification_binary: 2.1267 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1425 | Regression_eta: 0.0263 | Regression_pt: 0.1093 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6069 | Classification_binary: 2.1267 | Regression_cos_phi: 2.1452 | Regression_energy: 2.9199 | Regression_eta: 8.4939 | Regression_pt: 2.2074 | Regression_sin_phi: 2.0884 | LR: 2.63e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0257 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3044.4 | uss_MB: 1781.4 | child_rss_MB: 23103.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:38:42,564] INFO : Running validation on rank0 +rank0 [2026-11-15 04:38:42,564] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:38:42,564] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:38:42,670] INFO : saved batch0_epoch11000.parquet +rank0 [2026-11-15 04:38:42,750] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:38:42,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:38:42,877] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:38:42,923] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:38:42,968] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:38:43,013] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:38:43,059] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:38:43,132] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:38:43,209] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:38:43,284] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:38:43,331] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:38:43,376] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:38:43,422] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:38:43,467] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:38:43,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:38:43,575] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:38:43,579] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:38:43,581] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:38:43,584] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:38:43,584] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:38:43,586] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:38:43,587] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:38:43,589] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:38:43,590] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:38:43,592] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:38:43,593] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:38:43,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:38:43,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:38:43,598] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:38:43,600] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:38:43,604] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:38:44,567] INFO : VALIDATION | Step=11000/500000 | Train Loss=21.5883 | Valid Loss=14.4309 | Train Loss Unweighted=2.4287 | Valid Loss Unweighted=2.2411 | Stale=10 | ETA=666.5m +rank0 [2026-11-15 04:38:44,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:38:44,879] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:38:44,883] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:44,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:38:45,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:38:45,534] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:38:45,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:38:45,862] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:38:46,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:38:46,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:38:46,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:38:46,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:38:46,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:38:46,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:38:46,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:38:47,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:38:47,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:38:47,481] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:38:47,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:38:47,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:38:47,876] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:38:47,883] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,269] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_11000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:38:48,501] INFO : Plotted number of elements +rank0 [2026-11-15 04:38:49,442] INFO : Plotted jets +rank0 [2026-11-15 04:38:50,618] INFO : Plotted jet ratio +rank0 [2026-11-15 04:39:08,195] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-15 04:39:08,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:39:49,169] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-15 04:39:49,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:40:04,133] INFO : Step 12000/500000 rank0 | Train Loss: 10.2180 | Train Loss Unweighted: 2.0239 | Train Loss Components: Classification: 0.0109 | Classification_binary: 1.8560 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0752 | Regression_eta: 0.0036 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4378 | Classification_binary: 1.8560 | Regression_cos_phi: 1.5490 | Regression_energy: 1.5411 | Regression_eta: 1.1692 | Regression_pt: 1.4203 | Regression_sin_phi: 1.2446 | LR: 2.75e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7487s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0825 | max_tensorboard_logging: 0.2207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3053.5 | uss_MB: 1790.5 | child_rss_MB: 23105.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:40:04,135] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-12000.pth +rank0 [2026-11-15 04:40:04,183] INFO : Running validation on rank0 +rank0 [2026-11-15 04:40:04,183] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:40:04,183] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:40:04,285] INFO : saved batch0_epoch12000.parquet +rank0 [2026-11-15 04:40:04,365] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:40:04,443] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:40:04,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:40:04,536] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:40:04,581] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:40:04,627] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:40:04,673] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:40:04,745] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:40:04,823] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:40:04,899] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:40:04,946] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:40:04,992] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:40:05,038] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:40:05,083] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:40:05,129] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:40:05,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:40:05,197] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:40:05,198] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:40:05,201] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:40:05,202] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:40:05,203] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:40:05,206] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:40:05,206] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:40:05,208] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:40:05,210] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:40:05,211] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:40:05,212] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:40:05,215] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:40:05,216] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:40:05,218] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:40:05,219] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:40:06,154] INFO : VALIDATION | Step=12000/500000 | Train Loss=10.2180 | Valid Loss=14.5756 | Train Loss Unweighted=2.0239 | Valid Loss Unweighted=2.2481 | Stale=11 | ETA=665.0m +rank0 [2026-11-15 04:40:06,155] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:40:06,277] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:40:06,282] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:06,329] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:40:06,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:40:06,938] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:40:07,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:40:07,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:40:07,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:40:07,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:40:07,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:40:07,918] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:40:08,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:40:08,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:40:08,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:40:08,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:40:08,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:40:08,893] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:40:09,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:40:09,205] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:40:09,281] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:40:09,288] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,664] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_12000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:40:09,893] INFO : Plotted number of elements +rank0 [2026-11-15 04:40:11,291] INFO : Plotted jets +rank0 [2026-11-15 04:40:12,008] INFO : Plotted jet ratio +rank0 [2026-11-15 04:40:39,347] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-15 04:40:39,347] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:20,998] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-15 04:41:20,998] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:41:27,196] INFO : Step 13000/500000 rank0 | Train Loss: 12.4893 | Train Loss Unweighted: 2.0903 | Train Loss Components: Classification: 0.0144 | Classification_binary: 1.8362 | Regression_cos_phi: 0.0038 | Regression_energy: 0.1128 | Regression_eta: 0.0044 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0038 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.8928 | Classification_binary: 1.8362 | Regression_cos_phi: 1.3571 | Regression_energy: 2.3123 | Regression_eta: 1.4283 | Regression_pt: 2.3225 | Regression_sin_phi: 1.3401 | LR: 2.88e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8134s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0808 | max_tensorboard_logging: 0.2525 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3087.5 | uss_MB: 1824.5 | child_rss_MB: 23104.2 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:41:27,197] INFO : Running validation on rank0 +rank0 [2026-11-15 04:41:27,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:41:27,197] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:41:27,307] INFO : saved batch0_epoch13000.parquet +rank0 [2026-11-15 04:41:27,384] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:41:27,461] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:41:27,511] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:41:27,557] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:41:27,602] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:41:27,649] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:41:27,695] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:41:27,767] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:41:27,843] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:41:27,919] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:41:27,966] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:41:28,011] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:41:28,056] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:41:28,102] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:41:28,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:41:28,211] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:41:28,215] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:41:28,217] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:41:28,219] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:41:28,220] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:41:28,222] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:41:28,223] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:41:28,225] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:41:28,226] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:41:28,228] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:41:28,229] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:41:28,231] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:41:28,232] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:41:28,234] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:41:28,235] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:41:28,237] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:41:29,193] INFO : VALIDATION | Step=13000/500000 | Train Loss=12.4893 | Valid Loss=14.5489 | Train Loss Unweighted=2.0903 | Valid Loss Unweighted=2.2405 | Stale=12 | ETA=664.4m +rank0 [2026-11-15 04:41:29,193] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:41:29,544] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:41:29,548] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:29,595] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:41:30,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:41:30,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:41:30,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:41:30,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:41:30,692] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:41:30,858] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:41:31,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:41:31,184] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:41:31,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:41:31,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:41:31,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:41:31,823] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:41:31,988] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:41:32,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:41:32,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:41:32,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:41:32,539] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:41:32,545] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:32,931] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_13000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:41:33,158] INFO : Plotted number of elements +rank0 [2026-11-15 04:41:34,104] INFO : Plotted jets +rank0 [2026-11-15 04:41:35,284] INFO : Plotted jet ratio +rank0 [2026-11-15 04:42:11,767] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-15 04:42:11,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:42:50,283] INFO : Step 14000/500000 rank0 | Train Loss: 11.2762 | Train Loss Unweighted: 2.2799 | Train Loss Components: Classification: 0.0154 | Classification_binary: 2.0703 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0915 | Regression_eta: 0.0024 | Regression_pt: 0.0929 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 2.0224 | Classification_binary: 2.0703 | Regression_cos_phi: 1.5166 | Regression_energy: 1.8751 | Regression_eta: 0.7874 | Regression_pt: 1.8767 | Regression_sin_phi: 1.1277 | LR: 3.02e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0807 | max_tensorboard_logging: 0.2415 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3114.6 | uss_MB: 1851.6 | child_rss_MB: 23105.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:42:50,285] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-14000.pth +rank0 [2026-11-15 04:42:50,334] INFO : Running validation on rank0 +rank0 [2026-11-15 04:42:50,334] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:42:50,334] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:42:50,437] INFO : saved batch0_epoch14000.parquet +rank0 [2026-11-15 04:42:50,520] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:42:50,597] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:42:50,647] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:42:50,694] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:42:50,741] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:42:50,786] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:42:50,833] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:42:50,906] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:42:50,984] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:42:51,060] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:42:51,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:42:51,152] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:42:51,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:42:51,244] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:42:51,288] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:42:51,354] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:42:51,358] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:42:51,360] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:42:51,362] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:42:51,363] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:42:51,365] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:42:51,366] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:42:51,368] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:42:51,369] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:42:51,371] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:42:51,372] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:42:51,374] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:42:51,375] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:42:51,377] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:42:51,378] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:42:51,380] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:42:52,343] INFO : VALIDATION | Step=14000/500000 | Train Loss=11.2762 | Valid Loss=14.6595 | Train Loss Unweighted=2.2799 | Valid Loss Unweighted=2.2518 | Stale=13 | ETA=663.8m +rank0 [2026-11-15 04:42:52,343] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:42:52,473] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:42:52,478] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:52,525] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:42:52,954] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:42:53,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:42:53,281] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:42:53,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:42:53,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:42:53,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:42:53,940] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:42:54,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:42:54,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:42:54,425] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:42:54,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:42:54,750] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:42:54,909] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:42:55,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:42:55,228] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:42:55,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:42:55,448] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:42:55,454] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:55,845] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_14000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:42:56,080] INFO : Plotted number of elements +rank0 [2026-11-15 04:42:57,006] INFO : Plotted jets +rank0 [2026-11-15 04:42:58,179] INFO : Plotted jet ratio +rank0 [2026-11-15 04:43:02,235] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-15 04:43:02,235] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:43:43,673] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-15 04:43:43,673] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:44:12,791] INFO : Step 15000/500000 rank0 | Train Loss: 11.8908 | Train Loss Unweighted: 1.9384 | Train Loss Components: Classification: 0.0125 | Classification_binary: 1.7427 | Regression_cos_phi: 0.0055 | Regression_energy: 0.0836 | Regression_eta: 0.0036 | Regression_pt: 0.0851 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6463 | Classification_binary: 1.7427 | Regression_cos_phi: 1.9994 | Regression_energy: 1.7132 | Regression_eta: 1.1734 | Regression_pt: 1.7186 | Regression_sin_phi: 1.8971 | LR: 3.17e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7614s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0021 | max_backward: 0.0266 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2439 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3143.6 | uss_MB: 1880.6 | child_rss_MB: 23108.0 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:44:12,792] INFO : Running validation on rank0 +rank0 [2026-11-15 04:44:12,792] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:44:12,792] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:44:12,892] INFO : saved batch0_epoch15000.parquet +rank0 [2026-11-15 04:44:12,972] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:44:13,049] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:44:13,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:44:13,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:44:13,191] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:44:13,238] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:44:13,284] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:44:13,356] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:44:13,433] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:44:13,510] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:44:13,557] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:44:13,602] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:44:13,648] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:44:13,694] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:44:13,740] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:44:13,817] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:44:13,822] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:44:13,823] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:44:13,825] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:44:13,827] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:44:13,828] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:44:13,830] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:44:13,831] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:44:13,833] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:44:13,834] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:44:13,836] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:44:13,837] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:44:13,839] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:44:13,842] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:44:13,842] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:44:13,844] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:44:14,775] INFO : VALIDATION | Step=15000/500000 | Train Loss=11.8908 | Valid Loss=14.6530 | Train Loss Unweighted=1.9384 | Valid Loss Unweighted=2.2557 | Stale=14 | ETA=662.7m +rank0 [2026-11-15 04:44:14,776] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:44:15,168] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:44:15,172] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:15,232] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:44:15,771] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:44:15,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:44:16,100] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:44:16,265] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:44:16,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:44:16,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:44:16,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:44:16,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:44:17,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:44:17,255] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:44:17,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:44:17,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:44:17,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:44:17,906] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:44:18,070] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:44:18,215] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:44:18,305] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:44:18,312] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:18,695] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_15000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:44:18,924] INFO : Plotted number of elements +rank0 [2026-11-15 04:44:19,844] INFO : Plotted jets +rank0 [2026-11-15 04:44:21,018] INFO : Plotted jet ratio +rank0 [2026-11-15 04:44:34,074] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-15 04:44:34,074] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:15,509] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-15 04:45:15,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:45:35,696] INFO : Step 16000/500000 rank0 | Train Loss: 11.8593 | Train Loss Unweighted: 2.1546 | Train Loss Components: Classification: 0.0130 | Classification_binary: 1.9358 | Regression_cos_phi: 0.0037 | Regression_energy: 0.0947 | Regression_eta: 0.0038 | Regression_pt: 0.0988 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7124 | Classification_binary: 1.9358 | Regression_cos_phi: 1.3505 | Regression_energy: 1.9403 | Regression_eta: 1.2094 | Regression_pt: 1.9967 | Regression_sin_phi: 1.7141 | LR: 3.33e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0021 | max_backward: 0.0261 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3163.9 | uss_MB: 1900.9 | child_rss_MB: 23108.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:45:35,698] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-16000.pth +rank0 [2026-11-15 04:45:35,749] INFO : Running validation on rank0 +rank0 [2026-11-15 04:45:35,749] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:45:35,749] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:45:35,850] INFO : saved batch0_epoch16000.parquet +rank0 [2026-11-15 04:45:35,930] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:45:36,008] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:45:36,057] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:45:36,102] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:45:36,147] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:45:36,193] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:45:36,239] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:45:36,311] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:45:36,387] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:45:36,465] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:45:36,511] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:45:36,557] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:45:36,602] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:45:36,648] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:45:36,694] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:45:36,758] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:45:36,762] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:45:36,765] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:45:36,766] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:45:36,769] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:45:36,770] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:45:36,772] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:45:36,774] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:45:36,774] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:45:36,777] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:45:36,778] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:45:36,779] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:45:36,782] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:45:36,783] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:45:36,784] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:45:36,786] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:45:37,727] INFO : VALIDATION | Step=16000/500000 | Train Loss=11.8593 | Valid Loss=14.7104 | Train Loss Unweighted=2.1546 | Valid Loss Unweighted=2.2570 | Stale=15 | ETA=661.8m +rank0 [2026-11-15 04:45:37,728] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:45:37,833] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:45:37,837] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:37,884] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:45:38,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:45:38,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:45:38,663] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:45:38,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:45:38,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:45:39,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:45:39,314] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:45:39,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:45:39,637] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:45:39,799] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:45:39,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:45:40,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:45:40,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:45:40,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:45:40,603] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:45:40,757] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:45:40,852] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:45:40,870] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:41,255] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_16000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:45:41,484] INFO : Plotted number of elements +rank0 [2026-11-15 04:45:42,844] INFO : Plotted jets +rank0 [2026-11-15 04:45:43,576] INFO : Plotted jet ratio +rank0 [2026-11-15 04:46:06,049] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-15 04:46:06,050] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:47,105] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-15 04:46:47,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:46:58,135] INFO : Step 17000/500000 rank0 | Train Loss: 13.6762 | Train Loss Unweighted: 1.9723 | Train Loss Components: Classification: 0.0141 | Classification_binary: 1.8067 | Regression_cos_phi: 0.0052 | Regression_energy: 0.0645 | Regression_eta: 0.0116 | Regression_pt: 0.0653 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.8498 | Classification_binary: 1.8067 | Regression_cos_phi: 1.8654 | Regression_energy: 1.3220 | Regression_eta: 3.7451 | Regression_pt: 1.3194 | Regression_sin_phi: 1.7678 | LR: 3.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8151s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0237 | max_loss: 0.0021 | max_backward: 0.0269 | max_optimizer_step: 0.0842 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3108.1 | uss_MB: 1845.1 | child_rss_MB: 23107.6 | allocated_MB: 170.4 | reserved_MB: 684.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 04:46:58,137] INFO : Running validation on rank0 +rank0 [2026-11-15 04:46:58,137] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:46:58,137] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:46:58,239] INFO : saved batch0_epoch17000.parquet +rank0 [2026-11-15 04:46:58,322] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:46:58,400] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:46:58,451] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:46:58,498] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:46:58,543] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:46:58,589] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:46:58,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:46:58,708] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:46:58,787] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:46:58,863] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:46:58,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:46:58,958] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:46:59,004] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:46:59,054] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:46:59,100] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:46:59,164] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:46:59,168] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:46:59,170] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:46:59,172] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:46:59,173] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:46:59,175] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:46:59,176] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:46:59,178] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:46:59,179] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:46:59,181] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:46:59,183] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:46:59,184] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:46:59,186] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:46:59,188] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:46:59,189] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:46:59,192] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:47:00,122] INFO : VALIDATION | Step=17000/500000 | Train Loss=13.6762 | Valid Loss=14.7181 | Train Loss Unweighted=1.9723 | Valid Loss Unweighted=2.2673 | Stale=16 | ETA=660.6m +rank0 [2026-11-15 04:47:00,122] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:47:00,234] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:47:00,241] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:00,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:47:00,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:47:00,960] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:47:01,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:47:01,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:47:01,446] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:47:01,605] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:47:01,766] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:47:01,926] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:47:02,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:47:02,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:47:02,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:47:02,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:47:02,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:47:02,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:47:03,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:47:03,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:47:03,291] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:47:03,298] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:03,682] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_17000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:47:03,910] INFO : Plotted number of elements +rank0 [2026-11-15 04:47:04,825] INFO : Plotted jets +rank0 [2026-11-15 04:47:06,014] INFO : Plotted jet ratio +rank0 [2026-11-15 04:47:37,451] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-15 04:47:37,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:18,936] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-15 04:48:18,936] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:48:20,976] INFO : Step 18000/500000 rank0 | Train Loss: 9.1876 | Train Loss Unweighted: 2.1047 | Train Loss Components: Classification: 0.0150 | Classification_binary: 1.9264 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0782 | Regression_eta: 0.0020 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0015 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.9754 | Classification_binary: 1.9264 | Regression_cos_phi: 0.9267 | Regression_energy: 1.6022 | Regression_eta: 0.6299 | Regression_pt: 1.6005 | Regression_sin_phi: 0.5266 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.9282s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0025 | max_backward: 0.0273 | max_optimizer_step: 0.0829 | max_tensorboard_logging: 0.3601 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3216.5 | uss_MB: 1953.5 | child_rss_MB: 23108.3 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:48:20,977] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-18000.pth +rank0 [2026-11-15 04:48:21,026] INFO : Running validation on rank0 +rank0 [2026-11-15 04:48:21,026] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:48:21,026] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:48:21,128] INFO : saved batch0_epoch18000.parquet +rank0 [2026-11-15 04:48:21,208] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:48:21,286] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:48:21,335] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:48:21,380] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:48:21,425] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:48:21,471] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:48:21,517] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:48:21,591] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:48:21,670] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:48:21,748] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:48:21,795] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:48:21,841] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:48:21,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:48:21,933] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:48:21,981] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:48:22,057] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:48:22,062] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:48:22,064] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:48:22,065] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:48:22,067] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:48:22,068] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:48:22,070] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:48:22,072] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:48:22,073] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:48:22,075] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:48:22,076] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:48:22,078] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:48:22,079] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:48:22,081] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:48:22,082] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:48:22,084] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:48:23,020] INFO : VALIDATION | Step=18000/500000 | Train Loss=9.1876 | Valid Loss=14.8718 | Train Loss Unweighted=2.1047 | Valid Loss Unweighted=2.2705 | Stale=17 | ETA=659.6m +rank0 [2026-11-15 04:48:23,021] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:48:23,347] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:48:23,353] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:23,416] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:48:23,917] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:48:24,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:48:24,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:48:24,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:48:24,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:48:24,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:48:24,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:48:25,049] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:48:25,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:48:25,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:48:25,548] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:48:25,711] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:48:25,874] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:48:26,037] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:48:26,200] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:48:26,344] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:48:26,421] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:48:26,428] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:26,812] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_18000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:48:27,041] INFO : Plotted number of elements +rank0 [2026-11-15 04:48:27,971] INFO : Plotted jets +rank0 [2026-11-15 04:48:29,173] INFO : Plotted jet ratio +rank0 [2026-11-15 04:49:09,813] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-15 04:49:09,813] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:49:43,835] INFO : Step 19000/500000 rank0 | Train Loss: 9.0579 | Train Loss Unweighted: 2.1113 | Train Loss Components: Classification: 0.0085 | Classification_binary: 1.9515 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0698 | Regression_eta: 0.0029 | Regression_pt: 0.0726 | Regression_sin_phi: 0.0030 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.1201 | Classification_binary: 1.9515 | Regression_cos_phi: 1.1078 | Regression_energy: 1.4299 | Regression_eta: 0.9229 | Regression_pt: 1.4664 | Regression_sin_phi: 1.0593 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0302 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3240.6 | uss_MB: 1977.6 | child_rss_MB: 23104.5 | allocated_MB: 170.4 | reserved_MB: 694.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:49:43,836] INFO : Running validation on rank0 +rank0 [2026-11-15 04:49:43,836] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:49:43,836] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:49:43,938] INFO : saved batch0_epoch19000.parquet +rank0 [2026-11-15 04:49:44,016] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:49:44,094] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:49:44,143] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:49:44,189] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:49:44,235] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:49:44,281] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:49:44,327] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:49:44,399] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:49:44,476] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:49:44,554] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:49:44,600] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:49:44,645] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:49:44,690] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:49:44,736] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:49:44,783] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:49:44,861] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:49:44,865] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:49:44,867] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:49:44,868] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:49:44,870] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:49:44,872] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:49:44,873] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:49:44,875] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:49:44,876] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:49:44,878] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:49:44,879] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:49:44,881] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:49:44,882] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:49:44,884] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:49:44,885] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:49:44,887] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:49:45,822] INFO : VALIDATION | Step=19000/500000 | Train Loss=9.0579 | Valid Loss=15.0260 | Train Loss Unweighted=2.1113 | Valid Loss Unweighted=2.2789 | Stale=18 | ETA=658.5m +rank0 [2026-11-15 04:49:45,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:49:46,148] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:49:46,152] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:46,216] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:49:46,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:49:46,836] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:49:46,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:49:47,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:49:47,317] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:49:47,478] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:49:47,640] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:49:47,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:49:47,958] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:49:48,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:49:48,280] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:49:48,439] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:49:48,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:49:48,773] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:49:48,937] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:49:49,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:49:49,176] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:49:49,183] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,567] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_19000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:49:49,796] INFO : Plotted number of elements +rank0 [2026-11-15 04:49:50,742] INFO : Plotted jets +rank0 [2026-11-15 04:49:51,914] INFO : Plotted jet ratio +rank0 [2026-11-15 04:50:00,040] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-15 04:50:00,040] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:50:40,830] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-15 04:50:40,831] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:51:06,049] INFO : Step 20000/500000 rank0 | Train Loss: 9.8564 | Train Loss Unweighted: 2.1169 | Train Loss Components: Classification: 0.0126 | Classification_binary: 1.9340 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0807 | Regression_eta: 0.0025 | Regression_pt: 0.0811 | Regression_sin_phi: 0.0033 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.6593 | Classification_binary: 1.9340 | Regression_cos_phi: 0.9774 | Regression_energy: 1.6539 | Regression_eta: 0.8131 | Regression_pt: 1.6377 | Regression_sin_phi: 1.1810 | LR: 4.07e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0267 | max_optimizer_step: 0.0820 | max_tensorboard_logging: 0.2975 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3178.8 | uss_MB: 1915.8 | child_rss_MB: 23111.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:51:06,051] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-20000.pth +rank0 [2026-11-15 04:51:06,099] INFO : Running validation on rank0 +rank0 [2026-11-15 04:51:06,099] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:51:06,099] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:51:06,201] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-15 04:51:06,284] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:51:06,364] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:51:06,414] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:51:06,459] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:51:06,505] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:51:06,552] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:51:06,598] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:51:06,670] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:51:06,748] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:51:06,825] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:51:06,872] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:51:06,918] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:51:06,963] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:51:07,008] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:51:07,053] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:51:07,129] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:51:07,134] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:51:07,135] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:51:07,137] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:51:07,138] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:51:07,140] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:51:07,142] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:51:07,143] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:51:07,145] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:51:07,146] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:51:07,148] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:51:07,149] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:51:07,151] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:51:07,152] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:51:07,154] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:51:07,155] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:51:08,118] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.8564 | Valid Loss=14.9859 | Train Loss Unweighted=2.1169 | Valid Loss Unweighted=2.2766 | Stale=19 | ETA=657.2m +rank0 [2026-11-15 04:51:08,118] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:51:08,390] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:51:08,397] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:08,445] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:51:08,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:51:09,051] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:51:09,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:51:09,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:51:09,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:51:09,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:51:09,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:51:10,024] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:51:10,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:51:10,370] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:51:10,531] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:51:10,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:51:10,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:51:11,003] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:51:11,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:51:11,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:51:11,446] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:51:11,453] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:11,838] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:51:12,067] INFO : Plotted number of elements +rank0 [2026-11-15 04:51:12,975] INFO : Plotted jets +rank0 [2026-11-15 04:51:14,172] INFO : Plotted jet ratio +rank0 [2026-11-15 04:51:31,617] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-15 04:51:31,617] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:13,295] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-15 04:52:13,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:52:29,148] INFO : Step 21000/500000 rank0 | Train Loss: 10.8404 | Train Loss Unweighted: 2.2876 | Train Loss Components: Classification: 0.0103 | Classification_binary: 2.1074 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0791 | Regression_eta: 0.0037 | Regression_pt: 0.0789 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3533 | Classification_binary: 2.1074 | Regression_cos_phi: 1.7273 | Regression_energy: 1.6205 | Regression_eta: 1.1942 | Regression_pt: 1.5941 | Regression_sin_phi: 1.2437 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.7995s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0321 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2611 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3290.5 | uss_MB: 2027.5 | child_rss_MB: 23108.7 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:52:29,150] INFO : Running validation on rank0 +rank0 [2026-11-15 04:52:29,150] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:52:29,150] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:52:29,253] INFO : saved batch0_epoch21000.parquet +rank0 [2026-11-15 04:52:29,334] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:52:29,411] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:52:29,461] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:52:29,507] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:52:29,553] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:52:29,600] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:52:29,646] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:52:29,718] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:52:29,796] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:52:29,873] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:52:29,919] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:52:29,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:52:30,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:52:30,056] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:52:30,102] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:52:30,166] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:52:30,170] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:52:30,172] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:52:30,173] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:52:30,175] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:52:30,176] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:52:30,178] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:52:30,180] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:52:30,181] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:52:30,183] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:52:30,184] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:52:30,186] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:52:30,187] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:52:30,189] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:52:30,190] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:52:30,192] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:52:31,134] INFO : VALIDATION | Step=21000/500000 | Train Loss=10.8404 | Valid Loss=15.1942 | Train Loss Unweighted=2.2876 | Valid Loss Unweighted=2.2919 | Stale=20 | ETA=656.2m +rank0 [2026-11-15 04:52:31,134] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:52:31,267] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:52:31,276] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:31,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:52:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:52:32,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:52:32,178] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:52:32,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:52:32,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:52:32,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:52:32,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:52:32,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:52:33,146] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:52:33,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:52:33,465] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:52:33,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:52:33,786] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:52:33,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:52:34,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:52:34,268] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:52:34,346] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:52:34,365] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:34,750] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_21000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:52:34,979] INFO : Plotted number of elements +rank0 [2026-11-15 04:52:35,898] INFO : Plotted jets +rank0 [2026-11-15 04:52:37,084] INFO : Plotted jet ratio +rank0 [2026-11-15 04:53:03,817] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-15 04:53:03,817] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:45,387] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-15 04:53:45,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:53:52,244] INFO : Step 22000/500000 rank0 | Train Loss: 8.8013 | Train Loss Unweighted: 1.8964 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.7365 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0689 | Regression_eta: 0.0025 | Regression_pt: 0.0707 | Regression_sin_phi: 0.0027 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7442 | Classification_binary: 1.7365 | Regression_cos_phi: 0.6952 | Regression_energy: 1.4115 | Regression_eta: 0.8193 | Regression_pt: 1.4280 | Regression_sin_phi: 0.9666 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0022 | max_backward: 0.0270 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2610 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3316.7 | uss_MB: 2053.7 | child_rss_MB: 23106.6 | allocated_MB: 170.4 | reserved_MB: 674.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:53:52,245] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-22000.pth +rank0 [2026-11-15 04:53:52,293] INFO : Running validation on rank0 +rank0 [2026-11-15 04:53:52,293] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:53:52,293] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:53:52,395] INFO : saved batch0_epoch22000.parquet +rank0 [2026-11-15 04:53:52,475] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:53:52,553] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:53:52,601] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:53:52,647] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:53:52,691] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:53:52,738] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:53:52,784] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:53:52,856] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:53:52,933] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:53:53,011] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:53:53,058] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:53:53,103] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:53:53,149] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:53:53,195] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:53:53,242] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:53:53,307] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:53:53,311] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:53:53,312] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:53:53,314] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:53:53,316] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:53:53,317] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:53:53,319] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:53:53,320] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:53:53,322] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:53:53,323] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:53:53,325] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:53:53,326] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:53:53,328] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:53:53,329] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:53:53,331] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:53:53,332] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:53:54,282] INFO : VALIDATION | Step=22000/500000 | Train Loss=8.8013 | Valid Loss=15.1373 | Train Loss Unweighted=1.8964 | Valid Loss Unweighted=2.2810 | Stale=21 | ETA=655.2m +rank0 [2026-11-15 04:53:54,282] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:53:54,413] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:53:54,421] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:54,469] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:53:55,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:53:55,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:53:55,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:53:55,533] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:53:55,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:53:55,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:53:56,022] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:53:56,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:53:56,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:53:56,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:53:56,664] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:53:56,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:53:56,990] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:53:57,163] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:53:57,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:53:57,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:53:57,568] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:53:57,574] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:57,963] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_22000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:53:58,195] INFO : Plotted number of elements +rank0 [2026-11-15 04:53:59,112] INFO : Plotted jets +rank0 [2026-11-15 04:54:00,302] INFO : Plotted jet ratio +rank0 [2026-11-15 04:54:36,188] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-15 04:54:36,188] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:55:15,327] INFO : Step 23000/500000 rank0 | Train Loss: 9.0959 | Train Loss Unweighted: 2.0937 | Train Loss Components: Classification: 0.0099 | Classification_binary: 1.9137 | Regression_cos_phi: 0.0024 | Regression_energy: 0.0821 | Regression_eta: 0.0031 | Regression_pt: 0.0805 | Regression_sin_phi: 0.0020 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3029 | Classification_binary: 1.9137 | Regression_cos_phi: 0.8760 | Regression_energy: 1.6837 | Regression_eta: 0.9853 | Regression_pt: 1.6271 | Regression_sin_phi: 0.7072 | LR: 4.73e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0022 | max_backward: 0.0267 | max_optimizer_step: 0.0819 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3325.3 | uss_MB: 2062.4 | child_rss_MB: 23109.6 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:55:15,329] INFO : Running validation on rank0 +rank0 [2026-11-15 04:55:15,329] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:55:15,329] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:55:15,431] INFO : saved batch0_epoch23000.parquet +rank0 [2026-11-15 04:55:15,513] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:55:15,591] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:55:15,641] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:55:15,687] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:55:15,733] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:55:15,780] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:55:15,826] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:55:15,898] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:55:15,976] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:55:16,052] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:55:16,099] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:55:16,145] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:55:16,191] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:55:16,237] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:55:16,283] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:55:16,347] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:55:16,351] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:55:16,353] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:55:16,354] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:55:16,356] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:55:16,358] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:55:16,359] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:55:16,361] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:55:16,362] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:55:16,364] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:55:16,365] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:55:16,367] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:55:16,368] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:55:16,370] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:55:16,372] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:55:16,373] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:55:17,332] INFO : VALIDATION | Step=23000/500000 | Train Loss=9.0959 | Valid Loss=15.2043 | Train Loss Unweighted=2.0937 | Valid Loss Unweighted=2.2998 | Stale=22 | ETA=654.1m +rank0 [2026-11-15 04:55:17,332] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:55:17,655] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:55:17,663] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:17,712] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:55:18,153] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:55:18,331] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:55:18,494] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:55:18,657] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:55:18,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:55:18,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:55:19,144] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:55:19,303] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:55:19,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:55:19,631] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:55:19,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:55:19,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:55:20,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:55:20,267] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:55:20,427] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:55:20,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:55:20,727] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:55:20,734] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:21,110] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_23000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:55:21,339] INFO : Plotted number of elements +rank0 [2026-11-15 04:55:22,254] INFO : Plotted jets +rank0 [2026-11-15 04:55:23,445] INFO : Plotted jet ratio +rank0 [2026-11-15 04:55:26,925] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-15 04:55:26,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:08,412] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-15 04:56:08,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:56:38,160] INFO : Step 24000/500000 rank0 | Train Loss: 9.6762 | Train Loss Unweighted: 2.0494 | Train Loss Components: Classification: 0.0133 | Classification_binary: 1.8872 | Regression_cos_phi: 0.0028 | Regression_energy: 0.0696 | Regression_eta: 0.0021 | Regression_pt: 0.0702 | Regression_sin_phi: 0.0042 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7535 | Classification_binary: 1.8872 | Regression_cos_phi: 1.0183 | Regression_energy: 1.4263 | Regression_eta: 0.6737 | Regression_pt: 1.4179 | Regression_sin_phi: 1.4993 | LR: 4.97e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7622s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0022 | max_backward: 0.0289 | max_optimizer_step: 0.0818 | max_tensorboard_logging: 0.2362 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3363.0 | uss_MB: 2100.0 | child_rss_MB: 23112.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:56:38,162] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-24000.pth +rank0 [2026-11-15 04:56:38,209] INFO : Running validation on rank0 +rank0 [2026-11-15 04:56:38,209] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:56:38,209] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:56:38,334] INFO : saved batch0_epoch24000.parquet +rank0 [2026-11-15 04:56:38,415] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:56:38,493] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:56:38,542] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:56:38,588] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:56:38,633] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:56:38,679] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:56:38,726] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:56:38,799] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:56:38,877] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:56:38,954] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:56:39,002] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:56:39,048] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:56:39,094] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:56:39,140] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:56:39,186] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:56:39,250] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:56:39,254] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:56:39,256] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:56:39,257] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:56:39,259] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:56:39,261] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:56:39,262] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:56:39,264] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:56:39,265] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:56:39,267] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:56:39,269] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:56:39,270] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:56:39,272] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:56:39,273] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:56:39,275] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:56:39,276] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:56:40,229] INFO : VALIDATION | Step=24000/500000 | Train Loss=9.6762 | Valid Loss=15.4143 | Train Loss Unweighted=2.0494 | Valid Loss Unweighted=2.3019 | Stale=23 | ETA=652.9m +rank0 [2026-11-15 04:56:40,230] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:56:40,562] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:56:40,570] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:40,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:56:41,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:56:41,237] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:56:41,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:56:41,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:56:41,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:56:41,886] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:56:42,048] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:56:42,206] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:56:42,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:56:42,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:56:42,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:56:42,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:56:43,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:56:43,165] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:56:43,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:56:43,470] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:56:43,544] INFO : Time taken to make predictions on device 0 is: 0.04 min +rank0 [2026-11-15 04:56:43,551] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:43,933] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_24000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:56:44,161] INFO : Plotted number of elements +rank0 [2026-11-15 04:56:45,094] INFO : Plotted jets +rank0 [2026-11-15 04:56:46,277] INFO : Plotted jet ratio +rank0 [2026-11-15 04:56:58,965] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-15 04:56:58,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:57:40,275] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-15 04:57:40,275] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:58:00,880] INFO : Step 25000/500000 rank0 | Train Loss: 8.5471 | Train Loss Unweighted: 2.1765 | Train Loss Components: Classification: 0.0110 | Classification_binary: 2.0188 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0692 | Regression_eta: 0.0024 | Regression_pt: 0.0710 | Regression_sin_phi: 0.0022 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4539 | Classification_binary: 2.0188 | Regression_cos_phi: 0.6718 | Regression_energy: 1.4181 | Regression_eta: 0.7667 | Regression_pt: 1.4354 | Regression_sin_phi: 0.7823 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8137s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0021 | max_backward: 0.0260 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.2378 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3381.7 | uss_MB: 2118.7 | child_rss_MB: 23111.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 04:58:00,881] INFO : Running validation on rank0 +rank0 [2026-11-15 04:58:00,881] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:58:00,881] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:58:00,986] INFO : saved batch0_epoch25000.parquet +rank0 [2026-11-15 04:58:01,068] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:58:01,147] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:58:01,196] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:58:01,242] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:58:01,286] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:58:01,332] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:58:01,378] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:58:01,450] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:58:01,529] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:58:01,606] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:58:01,653] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:58:01,699] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:58:01,744] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:58:01,790] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:58:01,835] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:58:01,900] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:58:01,909] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:58:01,912] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:58:01,914] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:58:01,915] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:58:01,917] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:58:01,919] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:58:01,920] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:58:01,921] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:58:01,923] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:58:01,924] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:58:01,926] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:58:01,929] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:58:01,929] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:58:01,931] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:58:01,933] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:58:02,875] INFO : VALIDATION | Step=25000/500000 | Train Loss=8.5471 | Valid Loss=15.6071 | Train Loss Unweighted=2.1765 | Valid Loss Unweighted=2.3078 | Stale=24 | ETA=651.6m +rank0 [2026-11-15 04:58:02,875] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:58:02,993] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:58:03,001] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:03,050] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:58:03,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:58:03,674] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:58:03,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:58:04,002] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:58:04,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:58:04,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:58:04,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:58:04,653] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:58:04,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:58:04,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:58:05,137] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:58:05,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:58:05,458] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:58:05,621] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:58:05,793] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:58:05,939] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:58:06,010] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:58:06,017] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:06,396] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_25000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:58:06,625] INFO : Plotted number of elements +rank0 [2026-11-15 04:58:07,540] INFO : Plotted jets +rank0 [2026-11-15 04:58:08,720] INFO : Plotted jet ratio +rank0 [2026-11-15 04:58:30,102] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-15 04:58:30,102] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:11,682] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-15 04:59:11,682] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 04:59:23,388] INFO : Step 26000/500000 rank0 | Train Loss: 8.8150 | Train Loss Unweighted: 2.0446 | Train Loss Components: Classification: 0.0083 | Classification_binary: 1.9155 | Regression_cos_phi: 0.0027 | Regression_energy: 0.0544 | Regression_eta: 0.0045 | Regression_pt: 0.0560 | Regression_sin_phi: 0.0032 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.0910 | Classification_binary: 1.9155 | Regression_cos_phi: 0.9643 | Regression_energy: 1.1156 | Regression_eta: 1.4540 | Regression_pt: 1.1322 | Regression_sin_phi: 1.1425 | LR: 5.47e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.8049s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2612 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3408.8 | uss_MB: 2145.8 | child_rss_MB: 23108.2 | allocated_MB: 170.9 | reserved_MB: 692.0 | inactive_split_MB: 11.1 +rank0 [2026-11-15 04:59:23,390] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-26000.pth +rank0 [2026-11-15 04:59:23,440] INFO : Running validation on rank0 +rank0 [2026-11-15 04:59:23,441] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 04:59:23,441] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 04:59:23,545] INFO : saved batch0_epoch26000.parquet +rank0 [2026-11-15 04:59:23,626] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 04:59:23,708] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 04:59:23,756] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 04:59:23,802] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 04:59:23,848] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 04:59:23,895] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 04:59:23,942] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 04:59:24,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 04:59:24,093] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 04:59:24,173] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 04:59:24,220] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 04:59:24,267] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 04:59:24,314] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 04:59:24,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 04:59:24,408] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 04:59:24,484] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 04:59:24,488] INFO : plotted pt_target histogram +rank0 [2026-11-15 04:59:24,490] INFO : plotted pt_pred histogram +rank0 [2026-11-15 04:59:24,492] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 04:59:24,493] INFO : plotted eta_target histogram +rank0 [2026-11-15 04:59:24,495] INFO : plotted eta_pred histogram +rank0 [2026-11-15 04:59:24,496] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 04:59:24,498] INFO : plotted sphi_target histogram +rank0 [2026-11-15 04:59:24,499] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 04:59:24,501] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 04:59:24,503] INFO : plotted cphi_target histogram +rank0 [2026-11-15 04:59:24,504] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 04:59:24,506] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 04:59:24,507] INFO : plotted energy_target histogram +rank0 [2026-11-15 04:59:24,509] INFO : plotted energy_pred histogram +rank0 [2026-11-15 04:59:24,511] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 04:59:25,441] INFO : VALIDATION | Step=26000/500000 | Train Loss=8.8150 | Valid Loss=15.6086 | Train Loss Unweighted=2.0446 | Valid Loss Unweighted=2.3190 | Stale=25 | ETA=650.4m +rank0 [2026-11-15 04:59:25,441] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 04:59:25,571] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 04:59:25,579] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:25,643] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 04:59:26,213] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 04:59:26,381] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 04:59:26,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 04:59:26,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 04:59:26,877] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 04:59:27,035] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 04:59:27,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 04:59:27,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 04:59:27,515] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 04:59:27,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 04:59:27,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 04:59:28,014] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 04:59:28,175] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 04:59:28,335] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 04:59:28,498] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 04:59:28,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 04:59:28,733] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 04:59:28,740] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:29,122] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_26000/cld_edm_ttbar_pf +rank0 [2026-11-15 04:59:29,354] INFO : Plotted number of elements +rank0 [2026-11-15 04:59:30,769] INFO : Plotted jets +rank0 [2026-11-15 04:59:31,486] INFO : Plotted jet ratio +rank0 [2026-11-15 05:00:02,596] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-15 05:00:02,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:44,167] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-15 05:00:44,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:00:46,696] INFO : Step 27000/500000 rank0 | Train Loss: 10.7064 | Train Loss Unweighted: 2.3267 | Train Loss Components: Classification: 0.0135 | Classification_binary: 2.1281 | Regression_cos_phi: 0.0038 | Regression_energy: 0.0861 | Regression_eta: 0.0030 | Regression_pt: 0.0897 | Regression_sin_phi: 0.0025 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.7824 | Classification_binary: 2.1281 | Regression_cos_phi: 1.3589 | Regression_energy: 1.7643 | Regression_eta: 0.9536 | Regression_pt: 1.8127 | Regression_sin_phi: 0.9065 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8099s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0022 | max_backward: 0.0269 | max_optimizer_step: 0.0822 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3433.2 | uss_MB: 2170.2 | child_rss_MB: 23114.1 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:00:46,698] INFO : Running validation on rank0 +rank0 [2026-11-15 05:00:46,698] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:00:46,698] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:00:46,800] INFO : saved batch0_epoch27000.parquet +rank0 [2026-11-15 05:00:46,877] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:00:46,953] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:00:47,002] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:00:47,048] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:00:47,094] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:00:47,140] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:00:47,185] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:00:47,258] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:00:47,334] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:00:47,410] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:00:47,457] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:00:47,503] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:00:47,548] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:00:47,594] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:00:47,640] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:00:47,716] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:00:47,720] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:00:47,722] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:00:47,724] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:00:47,725] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:00:47,727] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:00:47,728] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:00:47,730] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:00:47,731] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:00:47,733] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:00:47,735] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:00:47,736] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:00:47,738] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:00:47,739] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:00:47,741] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:00:47,742] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:00:48,678] INFO : VALIDATION | Step=27000/500000 | Train Loss=10.7064 | Valid Loss=15.7854 | Train Loss Unweighted=2.3267 | Valid Loss Unweighted=2.3279 | Stale=26 | ETA=649.2m +rank0 [2026-11-15 05:00:48,679] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:00:48,994] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:00:49,001] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:49,060] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:00:49,552] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:00:49,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:00:49,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:00:50,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:00:50,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:00:50,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:00:50,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:00:50,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:00:50,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:00:51,019] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:00:51,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:00:51,346] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:00:51,508] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:00:51,671] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:00:51,833] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:00:51,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:00:52,050] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:00:52,068] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:52,451] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_27000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:00:52,679] INFO : Plotted number of elements +rank0 [2026-11-15 05:00:53,597] INFO : Plotted jets +rank0 [2026-11-15 05:00:54,770] INFO : Plotted jet ratio +rank0 [2026-11-15 05:01:34,856] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-15 05:01:34,856] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:02:09,696] INFO : Step 28000/500000 rank0 | Train Loss: 9.9499 | Train Loss Unweighted: 2.1307 | Train Loss Components: Classification: 0.0103 | Classification_binary: 1.9821 | Regression_cos_phi: 0.0031 | Regression_energy: 0.0627 | Regression_eta: 0.0051 | Regression_pt: 0.0638 | Regression_sin_phi: 0.0036 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.3512 | Classification_binary: 1.9821 | Regression_cos_phi: 1.1031 | Regression_energy: 1.2855 | Regression_eta: 1.6386 | Regression_pt: 1.2894 | Regression_sin_phi: 1.3000 | LR: 6.01e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7693s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0279 | max_optimizer_step: 0.0810 | max_tensorboard_logging: 0.2323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3461.2 | uss_MB: 2198.2 | child_rss_MB: 23112.6 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 9.6 +rank0 [2026-11-15 05:02:09,698] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-28000.pth +rank0 [2026-11-15 05:02:09,746] INFO : Running validation on rank0 +rank0 [2026-11-15 05:02:09,746] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:02:09,746] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:02:09,849] INFO : saved batch0_epoch28000.parquet +rank0 [2026-11-15 05:02:09,930] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:02:10,008] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:02:10,058] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:02:10,104] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:02:10,149] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:02:10,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:02:10,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:02:10,314] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:02:10,391] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:02:10,468] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:02:10,514] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:02:10,560] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:02:10,606] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:02:10,652] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:02:10,698] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:02:10,763] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:02:10,768] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:02:10,769] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:02:10,771] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:02:10,772] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:02:10,774] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:02:10,776] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:02:10,777] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:02:10,779] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:02:10,780] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:02:10,782] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:02:10,783] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:02:10,785] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:02:10,786] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:02:10,788] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:02:10,789] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:02:11,726] INFO : VALIDATION | Step=28000/500000 | Train Loss=9.9499 | Valid Loss=15.8502 | Train Loss Unweighted=2.1307 | Valid Loss Unweighted=2.3353 | Stale=27 | ETA=648.1m +rank0 [2026-11-15 05:02:11,727] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:02:11,855] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:02:11,863] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:11,932] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:02:12,401] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:02:12,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:02:12,742] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:02:12,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:02:13,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:02:13,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:02:13,396] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:02:13,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:02:13,718] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:02:13,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:02:14,043] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:02:14,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:02:14,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:02:14,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:02:14,702] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:02:14,847] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:02:14,944] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:02:14,951] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:15,338] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_28000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:02:15,569] INFO : Plotted number of elements +rank0 [2026-11-15 05:02:16,504] INFO : Plotted jets +rank0 [2026-11-15 05:02:17,700] INFO : Plotted jet ratio +rank0 [2026-11-15 05:02:25,511] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-15 05:02:25,511] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:06,147] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-15 05:03:06,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:03:31,644] INFO : Step 29000/500000 rank0 | Train Loss: 10.3286 | Train Loss Unweighted: 2.1904 | Train Loss Components: Classification: 0.0091 | Classification_binary: 2.0132 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0777 | Regression_eta: 0.0027 | Regression_pt: 0.0792 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.2011 | Classification_binary: 2.0132 | Regression_cos_phi: 1.2094 | Regression_energy: 1.5927 | Regression_eta: 0.8769 | Regression_pt: 1.6001 | Regression_sin_phi: 1.8354 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7964s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2615 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3488.2 | uss_MB: 2225.2 | child_rss_MB: 23109.3 | allocated_MB: 170.4 | reserved_MB: 672.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:03:31,646] INFO : Running validation on rank0 +rank0 [2026-11-15 05:03:31,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:03:31,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:03:31,752] INFO : saved batch0_epoch29000.parquet +rank0 [2026-11-15 05:03:31,833] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:03:31,912] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:03:31,962] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:03:32,008] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:03:32,054] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:03:32,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:03:32,151] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:03:32,223] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:03:32,301] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:03:32,380] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:03:32,427] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:03:32,473] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:03:32,519] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:03:32,565] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:03:32,612] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:03:32,678] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:03:32,683] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:03:32,684] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:03:32,686] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:03:32,688] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:03:32,689] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:03:32,691] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:03:32,693] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:03:32,694] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:03:32,696] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:03:32,697] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:03:32,699] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:03:32,700] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:03:32,702] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:03:32,704] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:03:32,705] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:03:33,643] INFO : VALIDATION | Step=29000/500000 | Train Loss=10.3286 | Valid Loss=15.9546 | Train Loss Unweighted=2.1904 | Valid Loss Unweighted=2.3392 | Stale=28 | ETA=646.6m +rank0 [2026-11-15 05:03:33,643] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:03:33,768] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:03:33,777] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:33,846] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:03:34,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:03:34,509] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:03:34,672] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:03:34,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:03:35,005] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:03:35,166] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:03:35,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:03:35,486] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:03:35,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:03:35,822] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:03:35,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:03:36,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:03:36,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:03:36,472] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:03:36,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:03:36,779] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:03:36,868] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:03:36,875] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:37,258] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_29000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:03:37,492] INFO : Plotted number of elements +rank0 [2026-11-15 05:03:38,426] INFO : Plotted jets +rank0 [2026-11-15 05:03:39,619] INFO : Plotted jet ratio +rank0 [2026-11-15 05:03:56,505] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-15 05:03:56,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:04:37,921] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-15 05:04:37,921] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:04:54,503] INFO : Step 30000/500000 rank0 | Train Loss: 10.5399 | Train Loss Unweighted: 2.0270 | Train Loss Components: Classification: 0.0111 | Classification_binary: 1.8312 | Regression_cos_phi: 0.0019 | Regression_energy: 0.0864 | Regression_eta: 0.0022 | Regression_pt: 0.0877 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.4651 | Classification_binary: 1.8312 | Regression_cos_phi: 0.6968 | Regression_energy: 1.7715 | Regression_eta: 0.7114 | Regression_pt: 1.7713 | Regression_sin_phi: 2.2925 | LR: 6.58e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.8446s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0022 | max_backward: 0.0271 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2646 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3511.9 | uss_MB: 2248.9 | child_rss_MB: 23103.4 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 11.6 +rank0 [2026-11-15 05:04:54,505] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/checkpoints/checkpoint-30000.pth +rank0 [2026-11-15 05:04:54,555] INFO : Running validation on rank0 +rank0 [2026-11-15 05:04:54,555] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:04:54,555] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:04:54,657] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-15 05:04:54,737] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:04:54,815] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:04:54,865] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:04:54,911] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:04:54,957] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:04:55,003] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:04:55,050] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:04:55,123] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:04:55,202] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:04:55,279] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:04:55,326] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:04:55,374] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:04:55,420] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:04:55,467] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:04:55,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:04:55,578] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:04:55,582] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:04:55,583] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:04:55,585] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:04:55,587] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:04:55,588] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:04:55,590] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:04:55,591] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:04:55,593] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:04:55,595] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:04:55,596] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:04:55,598] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:04:55,599] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:04:55,601] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:04:55,602] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:04:55,604] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:04:56,543] INFO : VALIDATION | Step=30000/500000 | Train Loss=10.5399 | Valid Loss=16.1117 | Train Loss Unweighted=2.0270 | Valid Loss Unweighted=2.3436 | Stale=29 | ETA=645.3m +rank0 [2026-11-15 05:04:56,543] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:04:56,925] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:04:56,941] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:04:57,008] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:04:57,510] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:04:57,680] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:04:57,848] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:04:58,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:04:58,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:04:58,329] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:04:58,489] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:04:58,648] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:04:58,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:04:58,975] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:04:59,148] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:04:59,310] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:04:59,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:04:59,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:04:59,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:04:59,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:05:00,017] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:05:00,024] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:00,400] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:05:00,632] INFO : Plotted number of elements +rank0 [2026-11-15 05:05:01,556] INFO : Plotted jets +rank0 [2026-11-15 05:05:02,767] INFO : Plotted jet ratio +rank0 [2026-11-15 05:05:28,800] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-15 05:05:28,801] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:10,369] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-15 05:06:10,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-15 05:06:17,735] INFO : Step 31000/500000 rank0 | Train Loss: 10.0723 | Train Loss Unweighted: 2.0259 | Train Loss Components: Classification: 0.0096 | Classification_binary: 1.8785 | Regression_cos_phi: 0.0026 | Regression_energy: 0.0623 | Regression_eta: 0.0054 | Regression_pt: 0.0627 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 131.6042 | Classification_binary: 1.0000 | Regression_cos_phi: 360.8239 | Regression_energy: 20.4964 | Regression_eta: 322.4061 | Regression_pt: 20.2037 | Regression_sin_phi: 357.1826 | Task Weighted Losses: Classification: 1.2658 | Classification_binary: 1.8785 | Regression_cos_phi: 0.9365 | Regression_energy: 1.2774 | Regression_eta: 1.7380 | Regression_pt: 1.2664 | Regression_sin_phi: 1.7097 | LR: 6.88e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.7980s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0022 | max_backward: 0.0279 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2535 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3449.6 | uss_MB: 2186.6 | child_rss_MB: 23105.0 | allocated_MB: 170.4 | reserved_MB: 692.0 | inactive_split_MB: 7.6 +rank0 [2026-11-15 05:06:17,737] INFO : Running validation on rank0 +rank0 [2026-11-15 05:06:17,737] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-15 05:06:17,737] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-15 05:06:17,840] INFO : saved batch0_epoch31000.parquet +rank0 [2026-11-15 05:06:17,924] INFO : plotted energy_elemtype1 +rank0 [2026-11-15 05:06:18,002] INFO : plotted pt_elemtype1 +rank0 [2026-11-15 05:06:18,054] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-15 05:06:18,101] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-15 05:06:18,148] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-15 05:06:18,196] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-15 05:06:18,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-15 05:06:18,314] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-15 05:06:18,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-15 05:06:18,471] INFO : plotted pt_elemtype2 +rank0 [2026-11-15 05:06:18,517] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-15 05:06:18,564] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-15 05:06:18,610] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-15 05:06:18,657] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-15 05:06:18,703] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-15 05:06:18,768] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-15 05:06:18,772] INFO : plotted pt_target histogram +rank0 [2026-11-15 05:06:18,773] INFO : plotted pt_pred histogram +rank0 [2026-11-15 05:06:18,775] INFO : plotted pt_ratio histogram +rank0 [2026-11-15 05:06:18,777] INFO : plotted eta_target histogram +rank0 [2026-11-15 05:06:18,778] INFO : plotted eta_pred histogram +rank0 [2026-11-15 05:06:18,780] INFO : plotted eta_ratio histogram +rank0 [2026-11-15 05:06:18,781] INFO : plotted sphi_target histogram +rank0 [2026-11-15 05:06:18,783] INFO : plotted sphi_pred histogram +rank0 [2026-11-15 05:06:18,784] INFO : plotted sphi_ratio histogram +rank0 [2026-11-15 05:06:18,786] INFO : plotted cphi_target histogram +rank0 [2026-11-15 05:06:18,787] INFO : plotted cphi_pred histogram +rank0 [2026-11-15 05:06:18,789] INFO : plotted cphi_ratio histogram +rank0 [2026-11-15 05:06:18,790] INFO : plotted energy_target histogram +rank0 [2026-11-15 05:06:18,792] INFO : plotted energy_pred histogram +rank0 [2026-11-15 05:06:18,794] INFO : plotted energy_ratio histogram +rank0 [2026-11-15 05:06:19,757] INFO : VALIDATION | Step=31000/500000 | Train Loss=10.0723 | Valid Loss=16.3077 | Train Loss Unweighted=2.0259 | Valid Loss Unweighted=2.3535 | Stale=30 | ETA=644.2m +rank0 [2026-11-15 05:06:19,757] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-15 05:06:19,872] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-15 05:06:19,880] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:19,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-15 05:06:20,404] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-15 05:06:20,569] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-15 05:06:20,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-15 05:06:20,905] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-15 05:06:21,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-15 05:06:21,229] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-15 05:06:21,392] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-15 05:06:21,553] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-15 05:06:21,713] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-15 05:06:21,878] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-15 05:06:22,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-15 05:06:22,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-15 05:06:22,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-15 05:06:22,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-15 05:06:22,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-15 05:06:22,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-15 05:06:22,909] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-15 05:06:22,915] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:23,309] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_finetuning/cld_hcal_30mm_seed12345_20261115_042336_111334/preds_step_31000/cld_edm_ttbar_pf +rank0 [2026-11-15 05:06:23,546] INFO : Plotted number of elements +rank0 [2026-11-15 05:06:24,481] INFO : Plotted jets +rank0 [2026-11-15 05:06:25,715] INFO : Plotted jet ratio +rank0 [2026-11-15 05:06:26,098] INFO : Stopping early after 30 validation cycles without loss improvement (patience=30) +rank0 [2026-11-15 05:06:26,098] INFO : Training completed. Total time on device 0: 42.684min diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..2222c36f86d8c37b541d218c1587e727d0eb38f9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20bb5040b28323eeca06cb8d74f7563d3dfed2f98d257dd84f736a4176ae4fa2 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-88000.pth b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-88000.pth new file mode 100644 index 0000000000000000000000000000000000000000..fb5a8d7cf3fc9cb9f80cfcec8d2082e49284992a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/checkpoints/checkpoint-88000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0c53ea47355aa68b0131afb8b8d51700c70a92ef7654ef918ad12ed616a88c7 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_1000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..d23a2d4f3deeb78ff80ae9f1f3cf0592907d1313 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.6666666666666666, "train": {"Classification_binary": 0.2859426438808441, "Classification": 0.0006185905076563358, "Regression_pt": 0.718842625617981, "Regression_eta": 0.08947744220495224, "Regression_sin_phi": 0.02077680267393589, "Regression_cos_phi": 0.0147237628698349, "Regression_energy": 0.7598661184310913, "Total": 4.671631336212158, "Total_unweighted": 1.8902479861862957}, "train_group": {}, "valid": {"Classification_binary": 0.3143434723218282, "Classification": 0.0007153427383551995, "Regression_pt": 0.5621771017710367, "Regression_eta": 0.0817992736895879, "Regression_sin_phi": 0.01601302996277809, "Regression_cos_phi": 0.017209865152835846, "Regression_energy": 0.599832812945048, "Total": 4.283300081888835, "Total_unweighted": 1.59209089858147, "metrics/particle/class_1/efficiency": 0.8696421384811401, "metrics/particle/class_1/pid_accuracy": 0.9920092821121216, "metrics/particle/class_2/efficiency": 0.05301379784941673, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.15711785852909088, "metrics/particle/class_3/pid_accuracy": 0.9438256621360779, "metrics/particle/class_4/efficiency": 0.8469387888908386, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.008849557489156723, "metrics/particle/count/bias_mean": -32.02864456176758, "metrics/particle/count/mae": 32.0625, "metrics/particle/count/prediction_mean": 47.53776168823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3312031626701355, "metrics/particle/event/energy_response_mean": 0.6775708198547363, "metrics/particle/event/met_abs_error": 15.754435539245605, "metrics/particle/event/scalar_pt_relative_abs_error": 0.28994253277778625, "metrics/particle/event/scalar_pt_response_mean": 0.7159116864204407, "metrics/particle/event/vector_pt_closure": 0.12071532756090164, "metrics/particle/matched/delta_eta_abs_mean": 0.010640101507306099, "metrics/particle/matched/delta_phi_abs_mean": 0.0037182553205639124, "metrics/particle/matched/delta_r_mean": 0.012073332443833351, "metrics/particle/matched/energy_relative_abs_mean": 0.07670636475086212, "metrics/particle/matched/pid_accuracy": 0.97018963098526, "metrics/particle/matched/pt_relative_abs_mean": 0.06281678378582001, "metrics/particle/matching/duplicate_fraction": 0.014571749605238438, "metrics/particle/matching/efficiency": 0.5264536142349243, "metrics/particle/matching/f1": 0.6591132879257202, "metrics/particle/matching/purity": 0.8811525702476501, "metrics/particle/matching/target_coverage_dr0p05": 0.5186803340911865, "metrics/particle/matching/target_coverage_dr0p10": 0.5414273142814636, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.4844125807285309, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5417709946632385}, "step_train_time": 1.2147176265716553, "step_valid_time": 9.336407964000045, "step_total_time": 245.42296934127808, "elapsed_training_loop_seconds": 245.42296934127808, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7868102483446114, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28717968986057574, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 594, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 493, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 430, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7103746397694525, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.82996632996633, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.765527950310559, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17003367003367004, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6195965417867435, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7239057239057239, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.6677018633540373, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7103746397694525, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8039673982946126, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28633416167676773, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1433, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1283, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1126, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8342002600780234, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.895324494068388, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.863682261864692, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.104675505931612, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7321196358907672, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7857641311933008, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7579939414338607, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8342002600780234, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7812144595841544, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.30438196613733237, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1066, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 930, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 787, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8072916666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8724202626641651, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8385933273219116, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1275797373358349, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6831597222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7382739212007504, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7096483318304779, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8072916666666666, "step_test_time": 20.469531636999932, "step_plot_time": 12.873871834000056, "step_validation_cycle_time": 43.124404639999966} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..c3a9f9468563598027f5491a96e5d1aec69f9b96 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 6.666666666666667, "train": {"Classification_binary": 0.26622045040130615, "Classification": 0.000483333453303203, "Regression_pt": 0.39492300152778625, "Regression_eta": 0.10347061604261398, "Regression_sin_phi": 0.014108027331531048, "Regression_cos_phi": 0.01341337338089943, "Regression_energy": 0.4160347580909729, "Total": 3.9012770652770996, "Total_unweighted": 1.208653560228413}, "train_group": {}, "valid": {"Classification_binary": 0.2626905043919881, "Classification": 0.0004697671780983607, "Regression_pt": 0.3571130037307739, "Regression_eta": 0.052342539032300316, "Regression_sin_phi": 0.013240377108256022, "Regression_cos_phi": 0.013178806751966476, "Regression_energy": 0.3725932836532593, "Total": 3.112918217976888, "Total_unweighted": 1.0716282818466425, "metrics/particle/class_1/efficiency": 0.8799709677696228, "metrics/particle/class_1/pid_accuracy": 0.9863240122795105, "metrics/particle/class_2/efficiency": 0.057734206318855286, "metrics/particle/class_2/pid_accuracy": 0.012578615918755531, "metrics/particle/class_3/efficiency": 0.3574146032333374, "metrics/particle/class_3/pid_accuracy": 0.951463520526886, "metrics/particle/class_4/efficiency": 0.8027210831642151, "metrics/particle/class_4/pid_accuracy": 0.04661016911268234, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.15789473056793213, "metrics/particle/count/bias_mean": -22.58203125, "metrics/particle/count/mae": 22.62109375, "metrics/particle/count/prediction_mean": 56.984375, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3196184039115906, "metrics/particle/event/energy_response_mean": 0.6824414730072021, "metrics/particle/event/met_abs_error": 15.383869171142578, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2673879563808441, "metrics/particle/event/scalar_pt_response_mean": 0.7369844317436218, "metrics/particle/event/vector_pt_closure": 0.11625389009714127, "metrics/particle/matched/delta_eta_abs_mean": 0.008784983307123184, "metrics/particle/matched/delta_phi_abs_mean": 0.004185140132904053, "metrics/particle/matched/delta_r_mean": 0.010702955536544323, "metrics/particle/matched/energy_relative_abs_mean": 0.08312705159187317, "metrics/particle/matched/pid_accuracy": 0.9651775360107422, "metrics/particle/matched/pt_relative_abs_mean": 0.07696489989757538, "metrics/particle/matching/duplicate_fraction": 0.026368705555796623, "metrics/particle/matching/efficiency": 0.6179815530776978, "metrics/particle/matching/f1": 0.7201800346374512, "metrics/particle/matching/purity": 0.862878143787384, "metrics/particle/matching/target_coverage_dr0p05": 0.6221218705177307, "metrics/particle/matching/target_coverage_dr0p10": 0.6450488567352295, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.548987865447998, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6373410820960999}, "step_train_time": 1.722330093383789, "step_valid_time": 9.67263203399989, "step_total_time": 2739.397389650345, "elapsed_training_loop_seconds": 2739.397389650345, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8073442060733769, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2738605088970435, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 580, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 510, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 461, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7348703170028819, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8793103448275862, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8006279434850864, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1206896551724138, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6642651296829971, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7948275862068965, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7237048665620094, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7348703170028819, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8504521908959681, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2684007459865516, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1306, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8491547464239272, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9000689179875948, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8738708598193375, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09993108201240523, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7782834850455137, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8249483115093039, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8009367681498829, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8491547464239272, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8163998658556125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28292517442698883, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1068, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 939, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 827, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8151041666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8792134831460674, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.845945945945946, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12078651685393259, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7178819444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7743445692883895, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7450450450450451, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8151041666666666, "step_test_time": 20.916766064999592, "step_plot_time": 12.702506512999662, "step_validation_cycle_time": 43.740771117999884} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_11000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..a1aa5012e706beadafa41b07d789410ace63177e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 7.333333333333333, "train": {"Classification_binary": 0.2612898051738739, "Classification": 0.0005172262317501009, "Regression_pt": 0.30965375900268555, "Regression_eta": 0.03983163833618164, "Regression_sin_phi": 0.009225037880241871, "Regression_cos_phi": 0.013025752268731594, "Regression_energy": 0.3284427225589752, "Total": 2.6454176902770996, "Total_unweighted": 0.9619859414524399}, "train_group": {}, "valid": {"Classification_binary": 0.26110051075617474, "Classification": 0.0004657705236847202, "Regression_pt": 0.3527195453643799, "Regression_eta": 0.050708899895350136, "Regression_sin_phi": 0.012877800812323889, "Regression_cos_phi": 0.012988549967606863, "Regression_energy": 0.3677489360173543, "Total": 3.049518267313639, "Total_unweighted": 1.0586100133368745, "metrics/particle/class_1/efficiency": 0.8833191394805908, "metrics/particle/class_1/pid_accuracy": 0.986054003238678, "metrics/particle/class_2/efficiency": 0.05954974517226219, "metrics/particle/class_2/pid_accuracy": 0.006097560748457909, "metrics/particle/class_3/efficiency": 0.3799741268157959, "metrics/particle/class_3/pid_accuracy": 0.9550460577011108, "metrics/particle/class_4/efficiency": 0.795918345451355, "metrics/particle/class_4/pid_accuracy": 0.05128205195069313, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.15789473056793213, "metrics/particle/count/bias_mean": -21.56770896911621, "metrics/particle/count/mae": 21.609375, "metrics/particle/count/prediction_mean": 57.99869918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31473061442375183, "metrics/particle/event/energy_response_mean": 0.6880486011505127, "metrics/particle/event/met_abs_error": 15.271782875061035, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2648065388202667, "metrics/particle/event/scalar_pt_response_mean": 0.739837110042572, "metrics/particle/event/vector_pt_closure": 0.11675090342760086, "metrics/particle/matched/delta_eta_abs_mean": 0.008665156550705433, "metrics/particle/matched/delta_phi_abs_mean": 0.004169205669313669, "metrics/particle/matched/delta_r_mean": 0.010580698028206825, "metrics/particle/matched/energy_relative_abs_mean": 0.0835312157869339, "metrics/particle/matched/pid_accuracy": 0.965683102607727, "metrics/particle/matched/pt_relative_abs_mean": 0.07747692614793777, "metrics/particle/matching/duplicate_fraction": 0.027815818786621094, "metrics/particle/matching/efficiency": 0.629469633102417, "metrics/particle/matching/f1": 0.7281590104103088, "metrics/particle/matching/purity": 0.8635475635528564, "metrics/particle/matching/target_coverage_dr0p05": 0.6351318359375, "metrics/particle/matching/target_coverage_dr0p10": 0.6565859913825989, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5580702424049377, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6481090784072876}, "step_train_time": 1.3814055919647217, "step_valid_time": 9.577262401000098, "step_total_time": 3015.584730863571, "elapsed_training_loop_seconds": 3015.584730863571, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8144355954591933, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2695610995316372, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 580, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 509, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7334293948126801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8775862068965518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7990580847723705, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12241379310344827, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7931034482758621, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7221350078492935, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7334293948126801, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8543320262006505, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2579078293997258, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1316, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1202, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8556566970091027, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.90633608815427, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8802675585284281, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09366391184573003, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7815344603381015, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8278236914600551, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8040133779264214, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8556566970091027, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8235299004927202, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28912263634867796, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1071, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 948, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 836, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8229166666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8851540616246498, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8529014844804319, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11484593837535013, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7256944444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.780578898225957, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7521367521367521, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8229166666666666, "step_test_time": 20.80343430999983, "step_plot_time": 12.803630018000149, "step_validation_cycle_time": 43.63301727399994} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_12000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..603c56cf4f66ca11eb15b016e9c2bc41221d5560 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 8.0, "train": {"Classification_binary": 0.2526642382144928, "Classification": 0.00043995966552756727, "Regression_pt": 0.3738826811313629, "Regression_eta": 0.03865470737218857, "Regression_sin_phi": 0.010885557159781456, "Regression_cos_phi": 0.01211786549538374, "Regression_energy": 0.39353570342063904, "Total": 2.7291555404663086, "Total_unweighted": 1.082180712459376}, "train_group": {}, "valid": {"Classification_binary": 0.2599717378616333, "Classification": 0.0004570154706016183, "Regression_pt": 0.3473341067632039, "Regression_eta": 0.05125008523464203, "Regression_sin_phi": 0.012926486631234487, "Regression_cos_phi": 0.012827008962631226, "Regression_energy": 0.36293192704518634, "Total": 3.0405295689900718, "Total_unweighted": 1.047698367969133, "metrics/particle/class_1/efficiency": 0.8835086226463318, "metrics/particle/class_1/pid_accuracy": 0.9859497547149658, "metrics/particle/class_2/efficiency": 0.05954974517226219, "metrics/particle/class_2/pid_accuracy": 0.006097560748457909, "metrics/particle/class_3/efficiency": 0.37639808654785156, "metrics/particle/class_3/pid_accuracy": 0.9558318257331848, "metrics/particle/class_4/efficiency": 0.7925170063972473, "metrics/particle/class_4/pid_accuracy": 0.06437768042087555, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.19298245012760162, "metrics/particle/count/bias_mean": -22.09635353088379, "metrics/particle/count/mae": 22.1328125, "metrics/particle/count/prediction_mean": 57.47005081176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.31740525364875793, "metrics/particle/event/energy_response_mean": 0.6849852204322815, "metrics/particle/event/met_abs_error": 15.195321083068848, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2656943202018738, "metrics/particle/event/scalar_pt_response_mean": 0.7386612296104431, "metrics/particle/event/vector_pt_closure": 0.11573615670204163, "metrics/particle/matched/delta_eta_abs_mean": 0.008502566255629063, "metrics/particle/matched/delta_phi_abs_mean": 0.0040619755163788795, "metrics/particle/matched/delta_r_mean": 0.010365563444793224, "metrics/particle/matched/energy_relative_abs_mean": 0.08289139717817307, "metrics/particle/matched/pid_accuracy": 0.9660465121269226, "metrics/particle/matched/pt_relative_abs_mean": 0.07647837698459625, "metrics/particle/matching/duplicate_fraction": 0.026077894493937492, "metrics/particle/matching/efficiency": 0.6280131340026855, "metrics/particle/matching/f1": 0.729276716709137, "metrics/particle/matching/purity": 0.8694745898246765, "metrics/particle/matching/target_coverage_dr0p05": 0.6341008543968201, "metrics/particle/matching/target_coverage_dr0p10": 0.653836727142334, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5570229291915894, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6455398201942444}, "step_train_time": 1.3949236869812012, "step_valid_time": 9.654074743000365, "step_total_time": 3292.338544368744, "elapsed_training_loop_seconds": 3292.338544368744, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8119596645282968, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26652465654800694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 511, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7363112391930836, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8780068728522337, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009404388714734, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12199312714776632, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7955326460481099, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.725705329153605, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7363112391930836, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8540007778114133, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2599135556622777, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1453, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1320, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1203, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8582574772431729, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9084652443220922, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8826479438314945, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09153475567790778, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.782184655396619, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8279421885753613, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8044132397191575, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8582574772431729, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8148495306852187, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28927554990486415, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 947, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 836, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8220486111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.87847866419295, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8493273542600897, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12152133580705009, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7256944444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7755102040816326, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7497757847533633, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8220486111111112, "step_test_time": 20.755772820999937, "step_plot_time": 12.916555470000276, "step_validation_cycle_time": 43.7732418669998} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_13000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..9022c735ea7a2d4a34d9e98ad522dda9c9028bd8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 8.666666666666666, "train": {"Classification_binary": 0.25963249802589417, "Classification": 0.0005183102330192924, "Regression_pt": 0.45674780011177063, "Regression_eta": 0.025955570861697197, "Regression_sin_phi": 0.012321176938712597, "Regression_cos_phi": 0.018009928986430168, "Regression_energy": 0.483832448720932, "Total": 3.1323115825653076, "Total_unweighted": 1.257017733878456}, "train_group": {}, "valid": {"Classification_binary": 0.2590150634447734, "Classification": 0.0004541149828583002, "Regression_pt": 0.34337206681569415, "Regression_eta": 0.04969557126363119, "Regression_sin_phi": 0.012696268657843271, "Regression_cos_phi": 0.012849275022745132, "Regression_energy": 0.3586518367131551, "Total": 3.000491460164388, "Total_unweighted": 1.0367341969007005, "metrics/particle/class_1/efficiency": 0.8856565356254578, "metrics/particle/class_1/pid_accuracy": 0.9850922226905823, "metrics/particle/class_2/efficiency": 0.06354393810033798, "metrics/particle/class_2/pid_accuracy": 0.0057142856530845165, "metrics/particle/class_3/efficiency": 0.40424561500549316, "metrics/particle/class_3/pid_accuracy": 0.9597214460372925, "metrics/particle/class_4/efficiency": 0.7993197441101074, "metrics/particle/class_4/pid_accuracy": 0.06382978707551956, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.18421052396297455, "metrics/particle/count/bias_mean": -20.49869728088379, "metrics/particle/count/mae": 20.52994728088379, "metrics/particle/count/prediction_mean": 59.06770706176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.306906133890152, "metrics/particle/event/energy_response_mean": 0.6956729888916016, "metrics/particle/event/met_abs_error": 15.047198295593262, "metrics/particle/event/scalar_pt_relative_abs_error": 0.25611934065818787, "metrics/particle/event/scalar_pt_response_mean": 0.7487325668334961, "metrics/particle/event/vector_pt_closure": 0.11617553234100342, "metrics/particle/matched/delta_eta_abs_mean": 0.00889482069760561, "metrics/particle/matched/delta_phi_abs_mean": 0.004160592798143625, "metrics/particle/matched/delta_r_mean": 0.010793383233249187, "metrics/particle/matched/energy_relative_abs_mean": 0.08520277589559555, "metrics/particle/matched/pid_accuracy": 0.9659853577613831, "metrics/particle/matched/pt_relative_abs_mean": 0.0781533494591713, "metrics/particle/matching/duplicate_fraction": 0.029560884460806847, "metrics/particle/matching/efficiency": 0.6413176655769348, "metrics/particle/matching/f1": 0.7361441254615784, "metrics/particle/matching/purity": 0.8638788461685181, "metrics/particle/matching/target_coverage_dr0p05": 0.6489109396934509, "metrics/particle/matching/target_coverage_dr0p10": 0.6700050830841064, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5650089383125305, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6600553393363953}, "step_train_time": 1.40327787399292, "step_valid_time": 9.627395958999841, "step_total_time": 3569.6847937107086, "elapsed_training_loop_seconds": 3569.6847937107086, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8179759850786874, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26412923318842196, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 587, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 513, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 468, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.739193083573487, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8739352640545145, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8009367681498829, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1260647359454855, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6743515850144092, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.797274275979557, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7306791569086651, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.739193083573487, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8706348096278234, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.24628893261647689, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1453, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1324, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8608582574772432, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9112181693048864, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8853226345703779, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08878183069511356, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7906371911573472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8368891947694426, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8131059846205283, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8608582574772432, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8266948701161256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29636164332088, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1082, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 956, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 853, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8298611111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8835489833641405, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.855863921217547, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11645101663585952, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7404513888888888, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.788354898336414, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7636526410026858, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8298611111111112, "step_test_time": 21.075924726000267, "step_plot_time": 12.830048506000367, "step_validation_cycle_time": 43.98846710199996} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_14000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..b6604584f58b7d0b07dc3e802beba16689c0a795 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 9.333333333333334, "train": {"Classification_binary": 0.27853143215179443, "Classification": 0.0005843833787366748, "Regression_pt": 0.34931474924087524, "Regression_eta": 0.027885418385267258, "Regression_sin_phi": 0.015529620461165905, "Regression_cos_phi": 0.012032595463097095, "Regression_energy": 0.3732590973377228, "Total": 2.914330244064331, "Total_unweighted": 1.0571372964186594}, "train_group": {}, "valid": {"Classification_binary": 0.25794529914855957, "Classification": 0.0004472250972564022, "Regression_pt": 0.3383215665817261, "Regression_eta": 0.049014141162236534, "Regression_sin_phi": 0.012591486175855001, "Regression_cos_phi": 0.012970654914776484, "Regression_energy": 0.3541601896286011, "Total": 2.9835427602132163, "Total_unweighted": 1.0254505627090111, "metrics/particle/class_1/efficiency": 0.8828453421592712, "metrics/particle/class_1/pid_accuracy": 0.9872987270355225, "metrics/particle/class_2/efficiency": 0.061365287750959396, "metrics/particle/class_2/pid_accuracy": 0.005917159840464592, "metrics/particle/class_3/efficiency": 0.3706916272640228, "metrics/particle/class_3/pid_accuracy": 0.9535098671913147, "metrics/particle/class_4/efficiency": 0.795918345451355, "metrics/particle/class_4/pid_accuracy": 0.07692307978868484, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.12389380484819412, "metrics/particle/count/bias_mean": -22.59505271911621, "metrics/particle/count/mae": 22.62630271911621, "metrics/particle/count/prediction_mean": 56.97135543823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32391583919525146, "metrics/particle/event/energy_response_mean": 0.6783459782600403, "metrics/particle/event/met_abs_error": 14.930123329162598, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2731866240501404, "metrics/particle/event/scalar_pt_response_mean": 0.7309513688087463, "metrics/particle/event/vector_pt_closure": 0.11559745669364929, "metrics/particle/matched/delta_eta_abs_mean": 0.00889320857822895, "metrics/particle/matched/delta_phi_abs_mean": 0.004322011023759842, "metrics/particle/matched/delta_r_mean": 0.010886216536164284, "metrics/particle/matched/energy_relative_abs_mean": 0.08206410706043243, "metrics/particle/matched/pid_accuracy": 0.966213047504425, "metrics/particle/matched/pt_relative_abs_mean": 0.07572146505117416, "metrics/particle/matching/duplicate_fraction": 0.025689080357551575, "metrics/particle/matching/efficiency": 0.6252965927124023, "metrics/particle/matching/f1": 0.7287743091583252, "metrics/particle/matching/purity": 0.8732916116714478, "metrics/particle/matching/target_coverage_dr0p05": 0.6285532116889954, "metrics/particle/matching/target_coverage_dr0p10": 0.6499582529067993, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5542409420013428, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6429541707038879}, "step_train_time": 1.3335278034210205, "step_valid_time": 9.594808139000634, "step_total_time": 3846.978466272354, "elapsed_training_loop_seconds": 3846.978466272354, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7979874008802599, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.26062618047724717, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 583, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 512, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 459, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7377521613832853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8782161234991424, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8018794048551292, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12178387650085763, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6613832853025937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7873070325900514, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7188723570869224, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7377521613832853, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8450379785217189, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25616098867605697, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1451, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1323, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1212, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8602080624187256, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9117849758787043, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8852459016393442, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08821502412129566, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.788036410923277, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8352860096485183, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8109735697557712, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8602080624187256, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8079911871948255, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2856078184915656, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1075, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 944, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 831, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8194444444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.878139534883721, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8477772788504715, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12186046511627907, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7213541666666666, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7730232558139535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7462954647507858, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8194444444444444, "step_test_time": 20.898320418000367, "step_plot_time": 12.71927992000019, "step_validation_cycle_time": 43.668174188000194} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_15000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..894074b8ec67e5fff11d1538611faf9c7e75bad4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 10.0, "train": {"Classification_binary": 0.24847403168678284, "Classification": 0.0003151947748847306, "Regression_pt": 0.3250861167907715, "Regression_eta": 0.03425556793808937, "Regression_sin_phi": 0.008854256011545658, "Regression_cos_phi": 0.012678426690399647, "Regression_energy": 0.3364616632461548, "Total": 2.4760007858276367, "Total_unweighted": 0.9661252571386285}, "train_group": {}, "valid": {"Classification_binary": 0.2568018039067586, "Classification": 0.0004443188275521, "Regression_pt": 0.3354020118713379, "Regression_eta": 0.048636471231778465, "Regression_sin_phi": 0.012424005816380182, "Regression_cos_phi": 0.012096965064605078, "Regression_energy": 0.3507818778355916, "Total": 2.911081314086914, "Total_unweighted": 1.016587454554004, "metrics/particle/class_1/efficiency": 0.8876780867576599, "metrics/particle/class_1/pid_accuracy": 0.9867274165153503, "metrics/particle/class_2/efficiency": 0.05991285294294357, "metrics/particle/class_2/pid_accuracy": 0.012121211737394333, "metrics/particle/class_3/efficiency": 0.37274596095085144, "metrics/particle/class_3/pid_accuracy": 0.9569299817085266, "metrics/particle/class_4/efficiency": 0.8061224222183228, "metrics/particle/class_4/pid_accuracy": 0.07594936341047287, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.18584071099758148, "metrics/particle/count/bias_mean": -22.51432228088379, "metrics/particle/count/mae": 22.55338478088379, "metrics/particle/count/prediction_mean": 57.05208206176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3208858370780945, "metrics/particle/event/energy_response_mean": 0.6813990473747253, "metrics/particle/event/met_abs_error": 15.377224922180176, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2705419361591339, "metrics/particle/event/scalar_pt_response_mean": 0.7339320778846741, "metrics/particle/event/vector_pt_closure": 0.11729437857866287, "metrics/particle/matched/delta_eta_abs_mean": 0.008276517502963543, "metrics/particle/matched/delta_phi_abs_mean": 0.003873201785609126, "metrics/particle/matched/delta_r_mean": 0.010045385919511318, "metrics/particle/matched/energy_relative_abs_mean": 0.07943420857191086, "metrics/particle/matched/pid_accuracy": 0.9669668674468994, "metrics/particle/matched/pt_relative_abs_mean": 0.07330630719661713, "metrics/particle/matching/duplicate_fraction": 0.026337411254644394, "metrics/particle/matching/efficiency": 0.6286677718162537, "metrics/particle/matching/f1": 0.732270359992981, "metrics/particle/matching/purity": 0.8767573237419128, "metrics/particle/matching/target_coverage_dr0p05": 0.6357045769691467, "metrics/particle/matching/target_coverage_dr0p10": 0.6537548899650574, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5583975911140442, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6447870135307312}, "step_train_time": 1.5544722080230713, "step_valid_time": 9.627699632999793, "step_total_time": 4123.142262935638, "elapsed_training_loop_seconds": 4123.142262935638, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8024490589049713, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2665101354591799, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 506, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 456, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.729106628242075, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8694158075601375, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7931034482758621, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13058419243986255, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6570605187319885, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7835051546391752, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7147335423197492, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.729106628242075, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8490417012917721, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2564790384674729, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1448, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1313, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.85370611183355, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9067679558011049, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8794373744139317, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09323204419889503, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7802340702210663, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8287292817679558, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8037508372404555, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.85370611183355, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8075493909807746, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28657526365469854, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 949, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 833, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8237847222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8803339517625232, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8511210762331839, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11966604823747681, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7230902777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7727272727272727, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.747085201793722, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8237847222222222, "step_test_time": 20.905406115000005, "step_plot_time": 13.018532994000452, "step_validation_cycle_time": 44.013420953000605} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_16000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..3d4037cf08b3e92f6e0df49d40a84da5374b1ec9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 10.666666666666666, "train": {"Classification_binary": 0.27034759521484375, "Classification": 0.0003978601598646492, "Regression_pt": 0.36323750019073486, "Regression_eta": 0.04170995205640793, "Regression_sin_phi": 0.007963189855217934, "Regression_cos_phi": 0.009588414803147316, "Regression_energy": 0.3957567811012268, "Total": 2.437727451324463, "Total_unweighted": 1.0890012933814432}, "train_group": {}, "valid": {"Classification_binary": 0.2560579975446065, "Classification": 0.00043976431091626483, "Regression_pt": 0.33174723386764526, "Regression_eta": 0.04934706290562948, "Regression_sin_phi": 0.012075026830037435, "Regression_cos_phi": 0.012290115157763163, "Regression_energy": 0.34782318274180096, "Total": 2.9037806193033853, "Total_unweighted": 1.0097803833583991, "metrics/particle/class_1/efficiency": 0.8886256814002991, "metrics/particle/class_1/pid_accuracy": 0.985390841960907, "metrics/particle/class_2/efficiency": 0.0631808266043663, "metrics/particle/class_2/pid_accuracy": 0.005747126415371895, "metrics/particle/class_3/efficiency": 0.40595754981040955, "metrics/particle/class_3/pid_accuracy": 0.9618592262268066, "metrics/particle/class_4/efficiency": 0.8027210831642151, "metrics/particle/class_4/pid_accuracy": 0.08898305147886276, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.1946902722120285, "metrics/particle/count/bias_mean": -20.53645896911621, "metrics/particle/count/mae": 20.57552146911621, "metrics/particle/count/prediction_mean": 59.02994918823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3102708160877228, "metrics/particle/event/energy_response_mean": 0.6925418972969055, "metrics/particle/event/met_abs_error": 15.232379913330078, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2593081295490265, "metrics/particle/event/scalar_pt_response_mean": 0.7453441023826599, "metrics/particle/event/vector_pt_closure": 0.11672741174697876, "metrics/particle/matched/delta_eta_abs_mean": 0.008395382203161716, "metrics/particle/matched/delta_phi_abs_mean": 0.004220759961754084, "metrics/particle/matched/delta_r_mean": 0.01037171483039856, "metrics/particle/matched/energy_relative_abs_mean": 0.08297494798898697, "metrics/particle/matched/pid_accuracy": 0.9670200943946838, "metrics/particle/matched/pt_relative_abs_mean": 0.07562094926834106, "metrics/particle/matching/duplicate_fraction": 0.02927098236978054, "metrics/particle/matching/efficiency": 0.6435760259628296, "metrics/particle/matching/f1": 0.7389376163482666, "metrics/particle/matching/purity": 0.8674754500389099, "metrics/particle/matching/target_coverage_dr0p05": 0.6503837704658508, "metrics/particle/matching/target_coverage_dr0p10": 0.6705778241157532, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5691983103752136, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6618717908859253}, "step_train_time": 2.708264112472534, "step_valid_time": 9.696001199999955, "step_total_time": 4402.048926830292, "elapsed_training_loop_seconds": 4402.048926830292, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8099840716749463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2794484839092469, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 519, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 470, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7478386167146974, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8871794871794871, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8115715402658327, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11282051282051282, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6772334293948127, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8034188034188035, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7349491790461298, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7478386167146974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8721523465940298, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2412838648448129, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1457, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1325, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1225, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8615084525357607, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9094028826355525, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8848080133555927, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09059711736444749, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7964889466840052, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8407687028140014, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8180300500834724, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8615084525357607, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8208862466231563, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.30094188990885773, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1084, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 960, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 851, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8333333333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8856088560885609, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8586762075134168, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11439114391143912, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7387152777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7850553505535055, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.761180679785331, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8333333333333334, "step_test_time": 21.310808412999904, "step_plot_time": 12.898995278999791, "step_validation_cycle_time": 44.358448797000165} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_17000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..23e8ee7306f971084bfe0ca4d69989dec8ea32d5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 11.333333333333334, "train": {"Classification_binary": 0.2596989870071411, "Classification": 0.0003218689817003906, "Regression_pt": 0.28743356466293335, "Regression_eta": 0.027091166004538536, "Regression_sin_phi": 0.010034749284386635, "Regression_cos_phi": 0.0060563357546925545, "Regression_energy": 0.29844069480895996, "Total": 2.035907745361328, "Total_unweighted": 0.8890773665043525}, "train_group": {}, "valid": {"Classification_binary": 0.2552570303281148, "Classification": 0.00043882231693714857, "Regression_pt": 0.32725417613983154, "Regression_eta": 0.04755823314189911, "Regression_sin_phi": 0.012456456820170084, "Regression_cos_phi": 0.01204753170410792, "Regression_energy": 0.3422415653864543, "Total": 2.8832003275553384, "Total_unweighted": 0.9972538158375148, "metrics/particle/class_1/efficiency": 0.8860355615615845, "metrics/particle/class_1/pid_accuracy": 0.9874157905578613, "metrics/particle/class_2/efficiency": 0.05846042186021805, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.38925662636756897, "metrics/particle/class_3/pid_accuracy": 0.9558248519897461, "metrics/particle/class_4/efficiency": 0.795918345451355, "metrics/particle/class_4/pid_accuracy": 0.07264957576990128, "metrics/particle/class_5/efficiency": 0.9824561476707458, "metrics/particle/class_5/pid_accuracy": 0.1785714328289032, "metrics/particle/count/bias_mean": -21.66015625, "metrics/particle/count/mae": 21.69400978088379, "metrics/particle/count/prediction_mean": 57.90625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.32330167293548584, "metrics/particle/event/energy_response_mean": 0.6789032816886902, "metrics/particle/event/met_abs_error": 15.145401954650879, "metrics/particle/event/scalar_pt_relative_abs_error": 0.27021896839141846, "metrics/particle/event/scalar_pt_response_mean": 0.7340574860572815, "metrics/particle/event/vector_pt_closure": 0.11704593151807785, "metrics/particle/matched/delta_eta_abs_mean": 0.008900088258087635, "metrics/particle/matched/delta_phi_abs_mean": 0.004504998214542866, "metrics/particle/matched/delta_r_mean": 0.010982927866280079, "metrics/particle/matched/energy_relative_abs_mean": 0.08168373256921768, "metrics/particle/matched/pid_accuracy": 0.967130720615387, "metrics/particle/matched/pt_relative_abs_mean": 0.07518185675144196, "metrics/particle/matching/duplicate_fraction": 0.028399892151355743, "metrics/particle/matching/efficiency": 0.6347881555557251, "metrics/particle/matching/f1": 0.7348052263259888, "metrics/particle/matching/purity": 0.8722342252731323, "metrics/particle/matching/target_coverage_dr0p05": 0.6397466659545898, "metrics/particle/matching/target_coverage_dr0p10": 0.6612663269042969, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5607050061225891, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6521347761154175}, "step_train_time": 1.5801246166229248, "step_valid_time": 9.603517505000127, "step_total_time": 4679.532653570175, "elapsed_training_loop_seconds": 4679.532653570175, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7996687096145821, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27161524448321805, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 582, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 514, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 466, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7406340057636888, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8831615120274914, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8056426332288401, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11683848797250859, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6714697406340058, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8006872852233677, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7304075235109718, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7406340057636888, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8547389799109224, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25461043888583557, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1448, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1314, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1210, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8543563068920677, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9074585635359116, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8801071667782987, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0925414364640884, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7867360208062418, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.835635359116022, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8104487608841259, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8543563068920677, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8116270468207912, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2915879059370078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1078, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 954, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 841, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.828125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8849721706864564, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8556053811659193, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1150278293135436, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7300347222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7801484230055659, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7542600896860987, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.828125, "step_test_time": 21.0762596630002, "step_plot_time": 12.928363727999567, "step_validation_cycle_time": 44.10327602600046} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_18000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..a39915529938f05c473b4911c4387a6ae71eef20 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 12.0, "train": {"Classification_binary": 0.2438691407442093, "Classification": 0.00041143441922031343, "Regression_pt": 0.36784085631370544, "Regression_eta": 0.027337612584233284, "Regression_sin_phi": 0.011657948605716228, "Regression_cos_phi": 0.00722066406160593, "Regression_energy": 0.37111184000968933, "Total": 2.309507369995117, "Total_unweighted": 1.0294494967383798}, "train_group": {}, "valid": {"Classification_binary": 0.2540850043296814, "Classification": 0.0004287535169472297, "Regression_pt": 0.3239232301712036, "Regression_eta": 0.048333207766215004, "Regression_sin_phi": 0.012321031341950098, "Regression_cos_phi": 0.012618290881315867, "Regression_energy": 0.33850125471750897, "Total": 2.910163243611654, "Total_unweighted": 0.9902107727248219, "metrics/particle/class_1/efficiency": 0.8877728581428528, "metrics/particle/class_1/pid_accuracy": 0.9874759912490845, "metrics/particle/class_2/efficiency": 0.05991285294294357, "metrics/particle/class_2/pid_accuracy": 0.012121211737394333, "metrics/particle/class_3/efficiency": 0.39530548453330994, "metrics/particle/class_3/pid_accuracy": 0.9599653482437134, "metrics/particle/class_4/efficiency": 0.795918345451355, "metrics/particle/class_4/pid_accuracy": 0.09829059988260269, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.1592920422554016, "metrics/particle/count/bias_mean": -21.15885353088379, "metrics/particle/count/mae": 21.18489646911621, "metrics/particle/count/prediction_mean": 58.40755081176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3174915313720703, "metrics/particle/event/energy_response_mean": 0.6855178475379944, "metrics/particle/event/met_abs_error": 15.244050025939941, "metrics/particle/event/scalar_pt_relative_abs_error": 0.26862284541130066, "metrics/particle/event/scalar_pt_response_mean": 0.7363706231117249, "metrics/particle/event/vector_pt_closure": 0.11538809537887573, "metrics/particle/matched/delta_eta_abs_mean": 0.008553987368941307, "metrics/particle/matched/delta_phi_abs_mean": 0.0043248324654996395, "metrics/particle/matched/delta_r_mean": 0.010579854249954224, "metrics/particle/matched/energy_relative_abs_mean": 0.08093402534723282, "metrics/particle/matched/pid_accuracy": 0.9682893753051758, "metrics/particle/matched/pt_relative_abs_mean": 0.07408295571804047, "metrics/particle/matching/duplicate_fraction": 0.02724212408065796, "metrics/particle/matching/efficiency": 0.6383720636367798, "metrics/particle/matching/f1": 0.736268937587738, "metrics/particle/matching/purity": 0.8696301579475403, "metrics/particle/matching/target_coverage_dr0p05": 0.6461943984031677, "metrics/particle/matching/target_coverage_dr0p10": 0.6655211448669434, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5668090581893921, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.656242311000824}, "step_train_time": 1.3695313930511475, "step_valid_time": 9.599046960000123, "step_total_time": 4956.206613302231, "elapsed_training_loop_seconds": 4956.206613302231, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.801114119595733, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.27589682507329805, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 585, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 512, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 463, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7377521613832853, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8752136752136752, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.800625488663018, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12478632478632479, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6671469740634006, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7914529914529914, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7240031274433151, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7377521613832853, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8517474755619089, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2546581541141564, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1450, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1314, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1207, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8543563068920677, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9062068965517242, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8795180722891566, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09379310344827586, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7847854356306893, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8324137931034483, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8078982597054887, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8543563068920677, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8106803408761494, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.28558428880469944, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1080, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 953, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 839, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8272569444444444, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8824074074074074, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8539426523297491, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1175925925925926, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7282986111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7768518518518519, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7517921146953405, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8272569444444444, "step_test_time": 21.151784262000547, "step_plot_time": 12.850696321000214, "step_validation_cycle_time": 43.9907304670005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_19000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..e5f9ce19954170dd7d58e620966d333bce56b189 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 12.666666666666666, "train": {"Classification_binary": 0.2522393763065338, "Classification": 0.0004614130302798003, "Regression_pt": 0.328625351190567, "Regression_eta": 0.08249638974666595, "Regression_sin_phi": 0.019896211102604866, "Regression_cos_phi": 0.009401633404195309, "Regression_energy": 0.36705851554870605, "Total": 3.6683526039123535, "Total_unweighted": 1.0601788903295528}, "train_group": {}, "valid": {"Classification_binary": 0.25342923402786255, "Classification": 0.0004302832918862502, "Regression_pt": 0.32270169258117676, "Regression_eta": 0.04784973959128062, "Regression_sin_phi": 0.012074310332536697, "Regression_cos_phi": 0.011996800700823465, "Regression_energy": 0.3377203146616618, "Total": 2.8504110972086587, "Total_unweighted": 0.9862023751872282, "metrics/particle/class_1/efficiency": 0.8871095180511475, "metrics/particle/class_1/pid_accuracy": 0.98661208152771, "metrics/particle/class_2/efficiency": 0.06172839552164078, "metrics/particle/class_2/pid_accuracy": 0.01764705963432789, "metrics/particle/class_3/efficiency": 0.4028760492801666, "metrics/particle/class_3/pid_accuracy": 0.9631727933883667, "metrics/particle/class_4/efficiency": 0.8095238208770752, "metrics/particle/class_4/pid_accuracy": 0.09663865715265274, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.22123894095420837, "metrics/particle/count/bias_mean": -21.54036521911621, "metrics/particle/count/mae": 21.56640625, "metrics/particle/count/prediction_mean": 58.02604293823242, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3185340166091919, "metrics/particle/event/energy_response_mean": 0.683638870716095, "metrics/particle/event/met_abs_error": 15.068602561950684, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2668943703174591, "metrics/particle/event/scalar_pt_response_mean": 0.7373921275138855, "metrics/particle/event/vector_pt_closure": 0.11583038419485092, "metrics/particle/matched/delta_eta_abs_mean": 0.008715873584151268, "metrics/particle/matched/delta_phi_abs_mean": 0.004312602337449789, "metrics/particle/matched/delta_r_mean": 0.010728765279054642, "metrics/particle/matched/energy_relative_abs_mean": 0.08034465461969376, "metrics/particle/matched/pid_accuracy": 0.9684661626815796, "metrics/particle/matched/pt_relative_abs_mean": 0.07152030616998672, "metrics/particle/matching/duplicate_fraction": 0.02656853012740612, "metrics/particle/matching/efficiency": 0.6414322257041931, "metrics/particle/matching/f1": 0.7418497204780579, "metrics/particle/matching/purity": 0.8795440196990967, "metrics/particle/matching/target_coverage_dr0p05": 0.6467998623847961, "metrics/particle/matching/target_coverage_dr0p10": 0.66668301820755, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.570098340511322, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.6577478647232056}, "step_train_time": 1.4828343391418457, "step_valid_time": 9.689386979000119, "step_total_time": 5233.177930831909, "elapsed_training_loop_seconds": 5233.177930831909, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8023919222012217, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2689461804456431, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 584, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 517, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 467, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7449567723342939, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8852739726027398, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.809076682316119, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11472602739726027, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6729106628242075, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7996575342465754, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.730829420970266, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7449567723342939, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8628783070062839, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2475137989618461, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1452, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1321, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1216, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8589076723016905, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9097796143250688, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8836120401337793, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09022038567493113, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7906371911573472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.837465564738292, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8133779264214047, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8589076723016905, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8183086014179741, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.29983313547401624, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 956, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 843, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8298611111111112, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8860055607043559, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8570147915732855, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11399443929564411, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7317708333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7812789620018535, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7557149260421335, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8298611111111112, "step_test_time": 21.279870012000174, "step_plot_time": 12.808473502000197, "step_validation_cycle_time": 44.23621908000041} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_2000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1eea3e74b64684c243d42f526fcf88233346e7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 1.3333333333333333, "train": {"Classification_binary": 0.26875701546669006, "Classification": 0.0006495531997643411, "Regression_pt": 0.4737890958786011, "Regression_eta": 0.102826327085495, "Regression_sin_phi": 0.013939326629042625, "Regression_cos_phi": 0.01676243357360363, "Regression_energy": 0.47095686197280884, "Total": 4.20642614364624, "Total_unweighted": 1.3476806138060056}, "train_group": {}, "valid": {"Classification_binary": 0.29263468583424884, "Classification": 0.0006035490659996867, "Regression_pt": 0.47572505474090576, "Regression_eta": 0.06800481677055359, "Regression_sin_phi": 0.01509506752093633, "Regression_cos_phi": 0.01591143508752187, "Regression_energy": 0.4932580391565959, "Total": 3.8027445475260415, "Total_unweighted": 1.361232648176762, "metrics/particle/class_1/efficiency": 0.8762437105178833, "metrics/particle/class_1/pid_accuracy": 0.9873111844062805, "metrics/particle/class_2/efficiency": 0.05482934042811394, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.24575820565223694, "metrics/particle/class_3/pid_accuracy": 0.9532507658004761, "metrics/particle/class_4/efficiency": 0.8741496801376343, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 1.0, "metrics/particle/class_5/pid_accuracy": 0.04385964944958687, "metrics/particle/count/bias_mean": -27.53125, "metrics/particle/count/mae": 27.56510353088379, "metrics/particle/count/prediction_mean": 52.03515625, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3284725844860077, "metrics/particle/event/energy_response_mean": 0.6753715872764587, "metrics/particle/event/met_abs_error": 15.588421821594238, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2854965627193451, "metrics/particle/event/scalar_pt_response_mean": 0.7191516757011414, "metrics/particle/event/vector_pt_closure": 0.11776804178953171, "metrics/particle/matched/delta_eta_abs_mean": 0.009344805963337421, "metrics/particle/matched/delta_phi_abs_mean": 0.003751457901671529, "metrics/particle/matched/delta_r_mean": 0.010923249647021294, "metrics/particle/matched/energy_relative_abs_mean": 0.07788167148828506, "metrics/particle/matched/pid_accuracy": 0.9662759304046631, "metrics/particle/matched/pt_relative_abs_mean": 0.06360995769500732, "metrics/particle/matching/duplicate_fraction": 0.017866525799036026, "metrics/particle/matching/efficiency": 0.5682327747344971, "metrics/particle/matching/f1": 0.6871079206466675, "metrics/particle/matching/purity": 0.8688787221908569, "metrics/particle/matching/target_coverage_dr0p05": 0.5671363472938538, "metrics/particle/matching/target_coverage_dr0p10": 0.588410496711731, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.520676851272583, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.5871503949165344}, "step_train_time": 1.2374444007873535, "step_valid_time": 9.621101502000101, "step_total_time": 521.982545375824, "elapsed_training_loop_seconds": 521.982545375824, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7732548174443514, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2928812939372223, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 595, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 522, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 455, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7521613832853026, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8773109243697479, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8099301784328937, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1226890756302521, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6556195965417867, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7647058823529411, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7059736229635376, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7521613832853026, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8230368373082664, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.285717299635472, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1445, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1307, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1148, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8498049414824448, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9044982698961938, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8762990278243379, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09550173010380623, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7464239271781534, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7944636678200692, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7696949379818974, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8498049414824448, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.7934490911793346, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2977295296645034, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1066, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 936, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 807, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8125, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8780487804878049, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8440036068530208, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12195121951219512, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7005208333333334, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7570356472795498, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7276825969341749, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8125, "step_test_time": 20.507689230999972, "step_plot_time": 12.648044620000064, "step_validation_cycle_time": 43.21073069399995} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..6af8c6bd3c0bc5e44da79f5d687297f90c9fba2b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_finetuning/idea_seed12345_20261115_021815_941856/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 13.333333333333334, "train": {"Classification_binary": 0.25039058923721313, "Classification": 0.00048305990640074015, "Regression_pt": 0.2582206428050995, "Regression_eta": 0.04368675872683525, "Regression_sin_phi": 0.01049773395061493, "Regression_cos_phi": 0.0038605709560215473, "Regression_energy": 0.2644806206226349, "Total": 2.147979974746704, "Total_unweighted": 0.83161997620482}, "train_group": {}, "valid": {"Classification_binary": 0.2526075839996338, "Classification": 0.00042131729423999786, "Regression_pt": 0.31669992208480835, "Regression_eta": 0.047324190537134804, "Regression_sin_phi": 0.011938845117886862, "Regression_cos_phi": 0.012048220882813135, "Regression_energy": 0.33109962940216064, "Total": 2.827293078104655, "Total_unweighted": 0.9721397093186775, "metrics/particle/class_1/efficiency": 0.8882466554641724, "metrics/particle/class_1/pid_accuracy": 0.9872692823410034, "metrics/particle/class_2/efficiency": 0.05628177151083946, "metrics/particle/class_2/pid_accuracy": 0.0, "metrics/particle/class_3/efficiency": 0.3954576551914215, "metrics/particle/class_3/pid_accuracy": 0.9611351490020752, "metrics/particle/class_4/efficiency": 0.8129251599311829, "metrics/particle/class_4/pid_accuracy": 0.08786610513925552, "metrics/particle/class_5/efficiency": 0.9912280440330505, "metrics/particle/class_5/pid_accuracy": 0.20353981852531433, "metrics/particle/count/bias_mean": -21.63541603088379, "metrics/particle/count/mae": 21.66666603088379, "metrics/particle/count/prediction_mean": 57.93098831176758, "metrics/particle/count/target_mean": 79.56640625, "metrics/particle/event/energy_relative_abs_error": 0.3214547336101532, "metrics/particle/event/energy_response_mean": 0.6810839176177979, "metrics/particle/event/met_abs_error": 15.061238288879395, "metrics/particle/event/scalar_pt_relative_abs_error": 0.2711535394191742, "metrics/particle/event/scalar_pt_response_mean": 0.7332574725151062, "metrics/particle/event/vector_pt_closure": 0.11609058827161789, "metrics/particle/matched/delta_eta_abs_mean": 0.008452543057501316, "metrics/particle/matched/delta_phi_abs_mean": 0.004327881149947643, "metrics/particle/matched/delta_r_mean": 0.01048983447253704, "metrics/particle/matched/energy_relative_abs_mean": 0.08052273839712143, "metrics/particle/matched/pid_accuracy": 0.9686082601547241, "metrics/particle/matched/pt_relative_abs_mean": 0.07255671918392181, "metrics/particle/matching/duplicate_fraction": 0.02620754763484001, "metrics/particle/matching/efficiency": 0.6386011242866516, "metrics/particle/matching/f1": 0.7390859723091125, "metrics/particle/matching/purity": 0.8770987391471863, "metrics/particle/matching/target_coverage_dr0p05": 0.6441978812217712, "metrics/particle/matching/target_coverage_dr0p10": 0.6641628742218018, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.5666290521621704, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.655064046382904}, "step_train_time": 1.4385433197021484, "step_valid_time": 9.746859107999626, "step_total_time": 5510.951695680618, "elapsed_training_loop_seconds": 5510.951695680618, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8019145075069604, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2646474773992198, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 694, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 580, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 509, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 460, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7334293948126801, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8775862068965518, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7990580847723705, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12241379310344827, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.6628242074927954, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7931034482758621, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7221350078492935, "step/idea_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7334293948126801, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8585011933024947, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.25261047282693516, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1538, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1442, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1319, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1211, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8576072821846554, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9147018030513177, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.885234899328859, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08529819694868239, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7873862158647594, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8398058252427184, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.812751677852349, "step/idea_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8576072821846554, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.8153772662853195, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2976645050413611, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1152, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1079, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 959, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 841, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8324652777777778, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8887859128822985, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8597041685342895, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11121408711770157, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7300347222222222, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7794253938832252, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7539220080681309, "step/idea_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8324652777777778, "step_test_time": 20.964748979999968, "step_plot_time": 12.817524721999689, "step_validation_cycle_time": 43.994320619000064} \ No newline at end of file